// #include "task_dag_graph.h"
#include <iostream>
#include <cassert>
#include <map>
#include <algorithm>
#include <sys/time.h>
#include <string>

#ifdef EMP_QUEUE
#include "args.h"
#endif

#include "aceMesh_clang.h"
#include "addr_area_manager.h"
#include "task_dag_graph.h"
#include "utils/acelog.h"
#include "aceMesh_utils.h"
#include "athread.h"

#include "am_machine.h"
#include "MemPool.h"

#ifdef THREAD_TRACE
extern "C"{
#include "thread_trace.h"
}
extern unsigned long begin_time;
extern unsigned long trace_time;
#endif

//#if defined( _CIRCULAR_QUEUE) || defined(MASTER) //||defined(MPI_SURPPORT)
#include "scheduler.h"
extern "C" void* my_malloc_aligned(int size);
#ifdef TARG_SW5
    #ifdef DISTRIBUTED_SCHEDULER
extern struct generic_scheduler schedulers[65];
    #else
extern struct generic_scheduler schedulers[2];
    #endif
#elif TARG_SW9
extern struct generic_scheduler* schedulers;
#endif
//#endif
#ifdef EMP_QUEUE
volatile unsigned long master_spawn=0;
extern volatile int eq_status;
#endif
#if defined(CONCURRENT_CONSTRUCT_GRAPH)&&defined(TARG_SW9)
__uncached volatile unsigned long con_status=0;
__uncached volatile unsigned long con_dest_ptr=0;
__uncached volatile int con_type;
volatile unsigned long con_graph;
__uncached volatile unsigned long con_src_ptr=0;
//volatile bool con_is_nei;
volatile int con_ref=0;
//#define multi_task 3
__uncached volatile unsigned long con_task_ptr[BLOCK_SIZE];
__uncached volatile unsigned long temp_task[MULTI_TASK];
__uncached volatile int con_sum_pre=0;
__uncached volatile int spawn_status[MULTI_STATUS];
__uncached volatile unsigned long con_task_num;
__uncached unsigned long block_time=0,block_temp=0;
__uncached volatile unsigned long main_task[BLOCK_SIZE],main_head,main_tail;
__uncached volatile unsigned long dest_task[MULTI_TASK],dest_head,dest_tail;
// aceMesh_task* dest, int type, tuple_rw_task& src, bool is_neighbor;
__uncached volatile unsigned long end_head,end_tail;
int init_queue=0;
#endif
#ifdef REUSE_GRAPH
extern int graph_reuse_times;
#endif

#ifdef ACEMESH_PROFILING_SUCC
extern unsigned long all_spawn_slave_task_time[65];
#endif
#if defined(SUCC_BATCH) && defined(CONCURRENT_CONSTRUCT_GRAPH) && (_SERIAL_QUEUE)
extern struct m2s_detail M2S_details_64[64];
extern struct m2s_detail M2S_detail_temp_64[64];
inline int get_affinity_id(struct task* self) 
{
    return self->affinity_id;
}
#endif

#if defined(SUCC_BATCH) && defined(CONCURRENT_CONSTRUCT_GRAPH) && (_SERIAL_QUEUE)
extern "C" void emp_d_master_consum_all();
#endif

#ifdef _CIRCULAR_QUEUE
extern volatile int __thread init_flag;
#endif
#ifdef _LDMQ
#ifdef TARG_SW9
unsigned long mqueue_empty_addr[64];
#else
extern int __thread mqueue_empty;
#endif
#endif

#ifdef ACEMESH_SCHEDULER_PROFILING
extern long long  total_vert_times;
extern int max_num_neigh;
#endif
#ifdef ACEMESH_PROFILING_CLASSIFY
#define MAX_PROF_RECORD 20
extern double classify_exec_time[MAX_PROF_RECORD];
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
extern unsigned long buil_prof[N_BUILD_PROF];
extern int first_graph_build;
#ifdef FINE_PROF2
extern unsigned long find_hash_ntimes1,find_hash_ntimes2;
#endif
extern unsigned long graph_build_start;
extern unsigned long graph_build_end;
    #ifdef USE_COMPOSITE_TASK
extern unsigned long total_start;
    #endif
#endif


#ifdef ACEMESH_PARA
extern unsigned long time_init; 
extern unsigned long exec_from; 
extern unsigned long exec_to; 
extern unsigned long tt_build; 
extern int gid3,interval; 

int first=1;
#endif
int val=0;
int spawn_num=0;
extern int cgid;
#ifdef LOCAL_FLAG
#ifdef TARG_SW5
extern volatile int __thread is_run;
#elif TARG_SW9
extern unsigned long is_run_addr[64];
#endif
#else
extern volatile int is_run;
#endif
extern int is_master;
extern int sche_num_threads;
extern int total_num_threads;

extern int my_mpi_rank;
//extern volatile int mpi_is_run;
extern void *cur_super_taskptr;

#ifdef EMP_QUEUE_PROFILING
extern unsigned max_length;
#endif

#ifdef CONCURRENT_CONSTRUCT_GRAPH
inline void set_task_state(int val, volatile int * state)
{
    *state = val;
    asm volatile("memb\n");
}
#endif

#ifdef EMP_MASTER
extern volatile int m2s_flag;
extern master_buff ready_succ_buff;
extern volatile master_buff m2s_pending;
extern "C" int master_buff_empty(master_buff * buff);
extern "C" int master_buff_push(master_buff * buff, struct task * t);
extern "C" void master_buff_consume();
extern "C" void EQ_main_push(struct task * t, int push_req);
#endif

namespace AceMesh_runtime {
#ifdef _EXEC_TIME
/*unsigned long rpcc()
{
  unsigned long time;
  asm("rtc %0" : "=&r"(time) ::"memory");
  return time;
}*/
struct timeval start;
struct timeval end;
double exec_time = 0.0;
unsigned long exec_count=0;
unsigned long last_exec_count=0;
int first_graph_spawn=1;
unsigned long start_count;
unsigned long end_count;
#endif

#ifdef _DAG_TIME
unsigned long dag_start_count;
unsigned long dag_end_count;
double dag_reuse_time = 0;
double dag_build_time = 0;
#endif

//#define decrement_ref_count(self) (--((struct task*)self)->ref_count_t)
inline void decrement_ref_count(struct task * self)
{
    --self->ref_count_t;
    asm volatile("memb\n");
}
#ifdef TARG_SW5
#define atomic_sub(_val_,_addr_)\
{       asm volatile("ldw_dec  %0, 0(%1)\n\t"  \
            :"=r"(_val_):"r"(_addr_) :"memory"); \
}
#define atomic_add(_val_,_addr_){\
  asm volatile("ldw_inc  %0,0(%1)\n\t"\
           :"=r"(_val_):"r"(_addr_):"memory");\
}

#define my_atomic_true(_val_,_addr_) \
{          asm volatile("ldw_set  %0,0(%1)\n\t"\
               :"=r"(_val_):"r"(_addr_):"memory");}
#elif TARG_SW9
#define atomic_add(_new_, _val_)                                    \
    {                                                                  \
        _new_ = __sync_val_compare_and_swap(&(_val_), _val_, ++_val_); \
    }
#endif

 /*
void my_mutex_lock(volatile int* my_lock)
{
#ifdef TARG_SW5

#ifdef MUTEX_BASIC
    unsigned int __addr,__tmpi;
    __asm__ __volatile__(
        "        memb\n"
        "        ldi      %1,%2\n"
        "1:      ldw_inc  %0,0(%1)\n"
        "        bne      %0,1b\n"
        "        memb\n"
      : "=&r"  (__tmpi),
        "=&r"  (__addr)
      : "m" (*my_lock)
      : "memory");
    return;
#else

    unsigned int __tmp = 0;
    unsigned int __cnt;
    __asm__ __volatile__ (
    "0:     ldw     %[__tmp], %[my_lock]\n"
    "       beq     %[__tmp], 2f\n"
    "       ldi     %[__cnt], 50\n"
    "       sll     %[__cnt], 4, %[__cnt]\n"
    "1:     subw    %[__cnt], 1, %[__cnt]\n"
    "       bne     %[__cnt], 1b\n"
    "       br      0b\n"
    "2:     ldw_inc    %[__tmp], %[my_lock]\n"
    "       bne     %[__tmp], 0b\n"
    "       memb    \n"
    "       br      3f\n"
    "3:     unop    \n"
    : [__tmp] "=&r"(__tmp), [__cnt] "=&r"(__cnt)
    : [my_lock] "m"(*(my_lock))
    : "memory");
    return;
//#endif
#elif TARG_SW9
    assert(0);
#endif
}
*/
/*
void my_mutex_unlock(volatile int* my_lock)
{
#ifdef TARG_SW5

#ifdef MUTEX_BASIC
    unsigned int __addr ;
    __asm__ __volatile__(
        "        memb\n"
        "        ldi      %0,%1\n"
        "        stw  $31,0(%0)\n"
        "        memb\n"
      : "=&r"  (__addr)
      : "m" (*my_lock)
      : "memory");
    return;
#else

    unsigned int __tmp = 0;
    __asm__ __volatile__ (
    //"       memb    \n"
    "       mov     0, %[__tmp]\n"
    "       stw     %[__tmp], %[my_lock]\n"
    "       memb      \n"
    : [__tmp] "=&r"(__tmp)
    : [my_lock] "m"(*(my_lock))
    : "memory"
    );
    return;
#endif
#elif TARG_SW9
    assert(0);
#endif
}
*/

#ifdef AUTO_PARTITION
task_dag_graph::task_dag_graph():my_disjoin_set(10240)
{
#ifdef DEBUG_GRAPH
    task_nums = 0;
    g_print_graph = false;
#endif
    sep_id = 0;

#ifdef ACEMESH_PERFORMANCE
    my_perf.thread_init();
#endif
}
#else
task_dag_graph::task_dag_graph()
{
#ifdef DEBUG_GRAPH
    task_nums = 0;
    g_print_graph = false;
#endif
    curr_dagNo = 0;

#ifdef ACEMESH_PERFORMANCE
    my_perf.thread_init();
#endif
}
#endif

struct task* compsite_end_task_constructor()
{
#ifdef MEMORY_POOL
#ifdef USE_SIMD
#ifdef TARG_SW5
    struct aceMesh_task* self = (aceMesh_task*)acemesh_myalloc_aligned_32(sizeof(aceMesh_task));
#elif TARG_SW9
    struct aceMesh_task* self = (aceMesh_task*)acemesh_myalloc_aligned_64(sizeof(aceMesh_task));
#endif
#else
    struct aceMesh_task* self = (aceMesh_task*)acemesh_myalloc_aligned_16(sizeof(aceMesh_task));
#endif
#else
#ifdef TARG_SW5
    struct aceMesh_task* self = (aceMesh_task*)malloc(sizeof(aceMesh_task));
#elif TARG_SW9
    //struct aceMesh_task* self = (aceMesh_task*)libc_uncached_malloc(sizeof(aceMesh_task));
    struct aceMesh_task* self = (aceMesh_task*)my_malloc_aligned(sizeof(aceMesh_task));
#endif
#endif
    //self->aceMesh_task_base = aceMesh_task_constructor();
    aceMesh_task_constructor(self);
    self->my_type = COMPOSITE_END_TASK;
    return &(self->task_base);
}

#ifdef DEBUG_GRAPH
struct end_task* end_task_constructor()
{
#ifdef MEMORY_POOL
#ifdef USE_SIMD
#ifdef TARG_SW5
    struct end_task* self = (end_task*)acemesh_myalloc_aligned_32(sizeof(end_task));
#elif TARG_SW9
    struct end_task* self = (end_task*)acemesh_myalloc_aligned_64(sizeof(end_task));
#endif
#else
    struct end_task* self = (end_task*)acemesh_myalloc_aligned_16(sizeof(end_task));
#endif
#else
#ifdef TARG_SW5
    struct end_task* self = (end_task*)malloc(sizeof(end_task));
#elif TARG_SW9
    //struct end_task* self = (end_task*)libc_uncached_malloc(sizeof(end_task));
    struct end_task* self = (end_task*)my_malloc_aligned(sizeof(end_task));
#endif
#endif
    //self->aceMesh_task_base = aceMesh_task_constructor();
    aceMesh_task_constructor(&(self->aceMesh_task_base));
    self->aceMesh_task_base.is_joint = 1;
    self->aceMesh_task_base.task_base.is_base_task = true;
    return self;
}
void end_task_destructor(struct end_task* p_end_task )
{
    aceMesh_task_destructor(&p_end_task->aceMesh_task_base);
}
#endif

task_dag_graph::~task_dag_graph()
{
    //if (my_mpi_rank <= 0 || my_mpi_rank == 64) { 
    if (my_mpi_rank <= 0 ) { 
#ifdef _EXEC_TIME  
      alog_info("rank: %d, exec time: %f", my_mpi_rank, (float)exec_count/MFREQ);
      printf("my_mpi_rank=%d,~exec time : %6.4f\n",my_mpi_rank,(float)exec_count/MFREQ);
#endif

#ifdef ACEMESH_SCHEDULER_PROFILING
      std::cout << "total vert times : " << total_vert_times << std::endl;
#endif
    }

}

void task_dag_graph::init(int thread_num,char* out_dir)
{
#ifdef AUTO_AFFINITY
    my_affinity.set_thread_num(thread_num);
#endif

    this->reset_task_graph();

#ifdef DEBUG_GRAPH
    graph_check.init(out_dir);
#endif
}
#ifdef AUTO_PARTITION
inline void task_dag_graph::union_set(aceMesh_task* src, aceMesh_task* dest)
{
    if(dest->get_group_id() < 0) 
    {
        dest->set_group_id( src->get_group_id() );
    }
    else
        dest->set_group_id( my_disjoin_set.union_set(src->get_group_id(), dest->get_group_id()) );
}
#endif

#ifdef REUSE_GRAPH
Error_Code task_dag_graph::dag_start(int dagNo, int *int_vec, int n1, double *float_vec, int n2) {
#ifdef _DAG_TIME
    if (dag_start_count == 0) {
        dag_start_count = rpcc();
    }
#endif
    curr_dagNo = dagNo;
    dag_reuse_flag[dagNo] = false;

    std::tr1::unordered_map<int, int>::iterator iter = dag_reuse_cnt.find(dagNo);
    if (iter != dag_reuse_cnt.end()) {

	std::vector<int>    vecI = dag_ivec_map[dagNo];
	std::vector<double> vecF = dag_fvec_map[dagNo];
        //compare vec length
        if (vecI.size() != n1 || vecF.size() != n2) {
            return 0;
        }

        for (int i = 0; i < n1; ++i) {
            if (vecI[i]!=int_vec[i])
                return 0;
        }
        for (int i = 0; i < n2; ++i) {
            if (vecF[i]!=float_vec[i])
                return 0;
        }
        iter->second++;
#ifdef ACEMESH_SCHEDULER_PROFILING
        if(my_mpi_rank<=0)
          std::cout << "reuse: "<< dagNo << "," << iter->second << std::endl;
#endif
        dag_reuse_flag[dagNo] = true;
        return 1;
    } else {
        //init
        dag_reuse_cnt[dagNo] = -1;
        for (int i = 0; i < n1; ++i) {
            dag_ivec_map[dagNo].push_back(int_vec[i]);
        }
        for (int i = 0; i < n2; ++i) {
            dag_fvec_map[dagNo].push_back(float_vec[i]);
        }
        return 0;
    }
}
#endif
unsigned long main_succ=0,main_temp=0;
int main_num=0;
inline int task_dag_graph::build_releationship
    (aceMesh_task* dest, int type, tuple_rw_task& src, bool is_neighbor) 
{


//printf("build_releation\n");

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    unsigned long relat_start=rpcc();
#endif

#ifdef THREAD_TRACE_LEVEL_1
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_build_releationship);
#endif 
#ifdef CONCURRENT_CONSTRUCT_GRAPH
    int ed1,ed2;   //carry return val of add or delete;
#endif     
    int res = 0;
    aceMesh_task* tmp = NULL;

#ifdef LIGHT_BUILD_GRAPH1


    if(is_neighbor) //if neighbor, must read, only RAW
    {
        if(src.w_task.t != NULL) 
        {
            tmp = src.w_task.t;
            if(type != INOUT_NONE) 
            {
#ifndef SPECIFY_END_TASKS
                    del_end_task(tmp);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif 
                    ed1 =add_successor(tmp,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                    res += ed1;
#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
                    add_successor(tmp,dest);

    #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
    #endif
                    ++res;
#endif
            }
        }
    }
    else            // if not neighbor, might read or write
    {
        //if(type == IN){
        if(src.w_task.t != NULL) 
        {
            tmp = src.w_task.t;
            if(type != INOUT_NONE) 
            {
    #ifndef SPECIFY_END_TASKS
                    del_end_task(tmp);
    #endif
    #ifdef CONCURRENT_CONSTRUCT_GRAPH
    #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
    #endif 
                    ed1 =add_successor(tmp,dest);
    #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
    #endif
                    res += ed1;
    #else
    #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
    #endif
                    add_successor(tmp,dest);

        #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
    #endif
                    ++res;
    #endif
                    
            }
        }//}
        //else   // if write
        //{
        for(std::vector<task_significance>::iterator r_task_itr = src.r_tasks.begin(); 
                r_task_itr != src.r_tasks.end(); ++r_task_itr)
                {
                    if(type != INOUT_NONE)
                    {
#ifndef SPECIFY_END_TASKS
                        del_end_task(r_task_itr->t);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
                        ed1 =add_successor(r_task_itr->t,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        res += ed1;

#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
                        add_successor(r_task_itr->t,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        ++res;
#endif                        
                            
                    }                   
                }
        //}

    }



#elif  LIGHT_BUILD_GRAPH2
    if(type != INOUT_NONE)
    {
        if(src.r_tasks.size() == 0)
        {
            if(src.w_task.t != NULL)
            {           
                tmp = src.w_task.t;
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
                    add_successor(tmp,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                    ++res;
#ifdef IMMEDIA_UPDATE
                    if(type==OUT || type==INOUT)
                    {
                        src.w_task.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                        src.w_task.significance = is_neighbor;
#endif
                    }
                    else if(type==IN)
                    {
                        task_significance t_s;
                        t_s.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                        t_s.significance = is_neighbor;
#endif 
                        src.r_tasks.push_back(t_s);
                    }
#endif
            }
        }
        else 
        {
            if(type == IN)
            {
                if(src.w_task.t != NULL)
                {
                    tmp = src.w_task.t;
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
                        add_successor(tmp,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        ++res;
                }
#ifdef IMMEDIA_UPDATE
                task_significance t_s;
                t_s.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                t_s.significance = is_neighbor;
#endif 
                src.r_tasks.push_back(t_s);
#endif
            }
            else 
            {
                for(std::vector<task_significance>::iterator r_task_itr = src.r_tasks.begin();  //why without WAW??？
                    r_task_itr != src.r_tasks.end(); ++r_task_itr)
                {                  
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
                    add_successor(r_task_itr->t,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                    ++res;
                }
#ifdef IMMEDIA_UPDATE
                src.w_task.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                src.w_task.significance = is_neighbor;
#endif
                src.r_tasks.clear();
#endif
            }
        }
    }
#elif  LIGHT_BUILD_GRAPH_SWITCH
            switch (type) {
                case IN:
                {
                    if (src.w_task.t!=NULL) {  //W-R
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
 #if defined(TARG_SW9)&& defined(AGENT_SUCC)
     //               printf("build\n");
                    /* 
                     if(my_h2ldm(con_status,63,cgid,int)==6)
                     {
                     int num=0;
                     my_h2ldm(con_status,63,cgid,int)=7;
                     asm volatile("memb\n"); 
                     while(num<2000){
                       add_successor(src.w_task.t,dest); 
                       num++;
                     }
                     wait_for_all((struct task *)dest);
                     
                     }
                     else{
                     */
                     my_h2ldm(con_dest_ptr,63,cgid,unsigned long)=(unsigned long)(src.w_task.t);
                     asm volatile("memb\n"); 
                     my_h2ldm(con_src_ptr,63,cgid,unsigned long)=(unsigned long)dest;
                     asm volatile("memb\n"); 
                     my_h2ldm(con_status,63,cgid,int)=6;
                     asm volatile("memb\n"); 
                   // int num=0;
                    //while(num<2000){
                    while(my_h2ldm(con_status,63,cgid,int));
  //                   }
                    // wait_for_all((struct task *)dest);
                     
 #else
                        main_temp=rpcc();
                        add_successor(src.w_task.t,dest);   //W空时，构造wR边
                        main_succ+=rpcc()-main_temp;
                        main_num++;
#endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        ++res;
                    }
#ifdef IMMEDIA_UPDATE
//push itself
                    task_significance t_s;
                    t_s.t = dest;
                    src.r_tasks.push_back(t_s);
#endif
                }
                    break;
                case OUT:
                case INOUT:
                {
                    if(src.r_tasks.size()!=0) {   //iterate build R-W, clear r_tasks
                        for(std::vector<task_significance>::iterator r_task_itr = src.r_tasks.begin();  //why without WAW??？
                            r_task_itr != src.r_tasks.end(); ++r_task_itr)
                        {                  
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
  #if defined(TARG_SW9)&&defined(AGENT_SUCC)
//                     
//                     con_src_ptr=(uint64_t)(r_task_itr->t);
//                     con_dest_ptr=(uint64_t)dest;
//                     con_status=1;
//                     while(con_status);
             //printf("build_1,status:%d\n",my_h2ldm(con_status,63,cgid,int));
/*
if(my_h2ldm(con_status,63,cgid,int)==6)
                      {
                          int num=0;
                             my_h2ldm(con_status,63,cgid,int)=7;
                               asm volatile("memb\n");
                               while(num<2000){
                               add_successor(r_task_itr->t,dest);
                                    num++;
                                     }
                                wait_for_all((struct task *)dest);
                                
                             }
                               else{
*/
my_h2ldm(con_dest_ptr,63,cgid,unsigned long)=(unsigned long)(r_task_itr->t);
                        asm volatile("memb\n");
                         my_h2ldm(con_src_ptr,63,cgid,unsigned long)=(unsigned long)dest;
                             asm volatile("memb\n");
                              my_h2ldm(con_status,63,cgid,int)=6;
                               asm volatile("memb\n");
                                 while(my_h2ldm(con_status,63,cgid,int));
//}
//wait_for_all((struct task *)dest);
 #else
                           main_temp=rpcc();
                            add_successor(r_task_itr->t,dest);
                            main_succ+=rpcc()-main_temp;
                            main_num++;
 #endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                            ++res;
                        }
#ifdef IMMEDIA_UPDATE
                        src.r_tasks.clear();
#endif
                    }
                    else {   //W-W
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
     #if defined(TARG_SW9)&&defined(AGENT_SUCC)
    // printf("build_2\n");
//                     con_src_ptr=(uint64_t)(src.w_task.t);
//                     con_dest_ptr=(uint64_t)dest;
//                     con_status=1;
//
//while(con_status);
/*
if(my_h2ldm(con_status,63,cgid,int)==6)               
{
                          int num=0;
                             my_h2ldm(con_status,63,cgid,int)=7;
                                asm volatile("memb\n");
                                while(num<2000){
                                     add_successor(src.w_task.t,dest);
                                         num++;
                                         }
                                            wait_for_all((struct task *)dest);
                                              
                                                 }
                                                else{
*/
my_h2ldm(con_dest_ptr,63,cgid,unsigned long)=(unsigned long)(src.w_task.t);
                        asm volatile("memb\n");
                          my_h2ldm(con_src_ptr,63,cgid,unsigned long)=(unsigned long)dest;
                            asm volatile("memb\n");
                             my_h2ldm(con_status,63,cgid,int)=6;
                               asm volatile("memb\n");
                                 while(my_h2ldm(con_status,63,cgid,int));

//}
//wait_for_all((struct task *)dest);
#else
                       main_temp=rpcc();
                        add_successor(src.w_task.t,dest);
                        main_succ+=rpcc()-main_temp;
                        main_num++;
 #endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        ++res;
                    }
//update w task
#ifdef IMMEDIA_UPDATE
                    src.w_task.t = dest;
#endif
                }
                    break;
                default:
                    break;
            }

#else


    if(src.r_tasks.size() == 0)
    {
        if(src.w_task.t != NULL)
        {
#ifdef CONCURRENT_CONSTRUCT_GRAPH
        //if(src.w_task.t->task_base.state<2)
        //{
#endif            
            tmp = src.w_task.t;
#ifdef LIGHT_BUILD_GRAPH
            if( !is_neighbor)
#else
            if( !is_neighbor && src.w_task.significance == 0)
#endif
            {
                if(type != INOUT_NONE)
                {
#ifndef SPECIFY_END_TASKS
                    del_end_task(tmp);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif 
#ifdef LIGHT_BUILD_GRAPH
                    ed1 =add_successor(tmp,dest);
#else
                    ed1 =set_vertical_task(tmp,dest);
#endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                    res += ed1;
                    /*
                    if (src.w_task.t->task_base.state==3)
                    {
                        if (dest->task_base.edge)
                        {
                            std::cout << "finished1 :" << src.w_task.t->task_base.state<< std::endl;
                            ed2 =del_successor(src.w_task.t,dest);
                            res -= ed2;
                        }
                    }
                    atomic_sub(val,&(dest->task_base.edge));
                    */
#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
#ifdef LIGHT_BUILD_GRAPH
                    add_successor(tmp,dest);
#else
                    set_vertical_task(tmp,dest);
#endif

  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                    ++res;
#endif
                
#ifdef AUTO_PARTITION
                    union_set(tmp, dest);
#endif

#ifdef IMMEDIA_UPDATE
                    if(type==OUT || type==INOUT)
                    {
                        src.w_task.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                        src.w_task.significance = is_neighbor;
#endif
                    }
                    else if(type==IN)
                    {
                        task_significance t_s;
                        t_s.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                        t_s.significance = is_neighbor;
#endif 
                        src.r_tasks.push_back(t_s);
                    }
#endif
                }
            }
#ifdef LIGHT_BUILD_GRAPH
            else if(is_neighbor)
#else
            else if(is_neighbor || src.w_task.significance == 1)
#endif
            {
                if(type != INOUT_NONE)
                {
#ifndef SPECIFY_END_TASKS
                    del_end_task(tmp);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH   
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_add_successor);
  #endif   
                    ed1=add_successor(tmp,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif 
                    res += ed1;
                    /*
                    if (src.w_task.t->task_base.state==3)
                    {
                        if (dest->task_base.edge)
                        {
                            std::cout << "res " << res << std::endl;
                            std::cout << "finished2 :" << src.w_task.t->task_base.state << std::endl;
                            ed2 =del_successor(src.w_task.t,dest);
                            res -= ed2;
                            std::cout << "res " << res << std::endl;
                        }
                    }
                    atomic_sub(val,&(dest->task_base.edge));
                    */
#else
#ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_add_successor);
#endif 
                    add_successor(tmp,dest);
#ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
#endif 
                    ++res;
#endif       

#ifdef AUTO_PARTITION
                    union_set(tmp, dest);
#endif
#ifdef IMMEDIA_UPDATE
                    if(type==OUT || type==INOUT)
                    {
                        src.w_task.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                        src.w_task.significance = is_neighbor;
#endif
                    }
                    else if(type==IN)
                    {
                        task_significance t_s;
                        t_s.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                        t_s.significance = is_neighbor;
#endif 
                        src.r_tasks.push_back(t_s);
                    }
#endif
                }
            }
#ifdef TEMP_ASSERT
            else 
            {
                assert(0);
            }
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
         //}//pair with if(finish) 
#endif            
        }
#ifdef TEMP_ASSERT
        else
        {
            assert(0);
        }
#endif
    }
    else 
    {
        if(type == IN)
        {
            if(src.w_task.t != NULL)
            {
#ifdef CONCURRENT_CONSTRUCT_GRAPH
            //if(src.w_task.t->task_base.state<2)
            //{
#endif 
                tmp = src.w_task.t;
#ifdef LIGHT_BUILD_GRAPH
                if(!is_neighbor)
#else
                if(!is_neighbor && src.w_task.significance == 0)
#endif
                {
                    if(type != INOUT_NONE)
                    {
#ifndef SPECIFY_END_TASKS
                        del_end_task(tmp);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
#ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
#endif 
#ifdef LIGHT_BUILD_GRAPH
                        ed1 =add_successor(tmp,dest);
#else
                        ed1 =set_vertical_task(tmp,dest);
#endif
#ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
#endif
                        res += ed1;
                        /*
                        if (src.w_task.t->task_base.state==3)
                        {
                            if (dest->task_base.edge)
                            {
                                std::cout << "finished3 :" << src.w_task.t->task_base.state << std::endl;
                                ed2 =del_successor(src.w_task.t,dest);
                                res -= ed2;
                            }
                        }
                        atomic_sub(val,&(dest->task_base.edge));
                        */
#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
#ifdef LIGHT_BUILD_GRAPH
                        add_successor(tmp,dest);
#else
                        set_vertical_task(tmp,dest);
#endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        ++res;
#endif

#ifdef AUTO_PARTITION
                        union_set(tmp, dest);
#endif
                    } 
                }  
#ifdef LIGHT_BUILD_GRAPH
                else if( is_neighbor)
#else
                else if( is_neighbor || src.w_task.significance == 1)
#endif
                {
                    if(type != INOUT_NONE)
                    {
#ifndef SPECIFY_END_TASKS
                        del_end_task(tmp);
#endif

#ifdef CONCURRENT_CONSTRUCT_GRAPH
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_add_successor);
  #endif
                        ed1 =add_successor(tmp,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        res += ed1;
                        /*
                        if (src.w_task.t->task_base.state==3)
                        {
                            if (dest->task_base.edge)
                            {
                                std::cout << "finished4 :" << src.w_task.t->task_base.state << std::endl;
                                ed2 = del_successor(src.w_task.t,dest);
                                res -= ed2;
                            }
                        }
                        atomic_sub(val,&(dest->task_base.edge));
                        */
#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_add_successor);
  #endif
                        add_successor(tmp,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                        ++res;
#endif                                
#ifdef AUTO_PARTITION
                    union_set(tmp, dest);
#endif
                    }
                }
#ifdef TEMP_ASSERT
            else 
            {
                assert(0);
            }
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
            //}//pair with if(finish) 
#endif
#ifdef IMMEDIA_UPDATE
                task_significance t_s;
                t_s.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
                t_s.significance = is_neighbor;
#endif 
                src.r_tasks.push_back(t_s);
#endif
            }
            else
            {
#ifdef THREAD_TRACE_LEVEL_1
            trace_time=rpcc()-begin_time;
            trace_print(trace_time,'0',RTS_event_end);
#endif
                return 0;
            }
        }
        else 
        {
            for(std::vector<task_significance>::iterator r_task_itr = src.r_tasks.begin();  //why without WAW??？
                r_task_itr != src.r_tasks.end(); ++r_task_itr)
                {
#ifdef CONCURRENT_CONSTRUCT_GRAPH
            //if(r_task_itr->t->task_base.state<2)
            //{
#endif                    
#ifdef LIGHT_BUILD_GRAPH
                    if(!is_neighbor)
#else
                    if(!is_neighbor && (*r_task_itr).significance == 0)
#endif
                    {
                        if(type != INOUT_NONE)
                        {
#ifndef SPECIFY_END_TASKS
                            del_end_task(r_task_itr->t);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
#ifdef LIGHT_BUILD_GRAPH
                            ed1 =add_successor(r_task_itr->t,dest);
#else
                            ed1 =set_vertical_task(r_task_itr->t,dest);
#endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                            res += ed1;
                        /*
                        if (r_task_itr->t->task_base.state==3)
                        {
                            if (dest->task_base.edge)
                            {
                                std::cout << "res1 :" << res << std::endl;
                                std::cout << "finished5 :" << r_task_itr->t->task_base.state << std::endl;
                                ed2 =del_successor( r_task_itr->t,dest);
                                res -= ed2;
                                std::cout << "res2 :" << res << std::endl;
                            }
                        }
                        atomic_sub(val,&(dest->task_base.edge));
                        */
#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_set_vertical_task);
  #endif
#ifdef LIGHT_BUILD_GRAPH
                            add_successor(r_task_itr->t,dest);
#else
                            set_vertical_task(r_task_itr->t,dest);
#endif
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                            ++res;
#endif                        
#ifdef AUTO_PARTITION
                        union_set(r_task_itr->t, dest);
#endif
                            
                        }
                    }  
#ifdef LIGHT_BUILD_GRAPH
                    else if( is_neighbor)
#else
                    else if( is_neighbor || (*r_task_itr).significance == 1)
#endif
                    {
                        if(type != INOUT_NONE)
                        {
#ifndef SPECIFY_END_TASKS
                            del_end_task(r_task_itr->t);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_add_successor);
  #endif
                            ed1 =add_successor(r_task_itr->t,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                            res += ed1;
                        /*
                        if (r_task_itr->t->task_base.state==3)
                        {
                            if (dest->task_base.edge)
                            {
                                std::cout << "res1 :" << res << std::endl;
                                std::cout << "finished6 :" << r_task_itr->t->task_base.state<< std::endl;
                                ed2 = del_successor(r_task_itr->t,dest);
                                res -= ed2;
                                std::cout << "res2 :" << res << std::endl;
                            }
                        }
                        atomic_sub(val,&(dest->task_base.edge));
                        */
#else
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_add_successor);
  #endif
                            add_successor(r_task_itr->t,dest);
  #ifdef THREAD_TRACE_LEVEL_1
                    trace_time=rpcc()-begin_time;
                    trace_print(trace_time,'0',RTS_event_end);
  #endif
                            ++res;
#endif
#ifdef AUTO_PARTITION
                            union_set(r_task_itr->t, dest);
#endif
                        }
                    }
#ifdef TEMP_ASSERT
                else 
                {
                    assert(0);
                }
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
            //}//pair with if(finish) 
#endif
                }
#ifdef IMMEDIA_UPDATE
            src.w_task.t=dest;
#if !defined(LIGHT_BUILD_GRAPH)
            src.w_task.significance = is_neighbor;
#endif
            src.r_tasks.clear();
#endif
        }
    }

#endif

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[BUILD_RELAT_IN]+=(rpcc()-relat_start);
#endif
#ifdef THREAD_TRACE_LEVEL_1
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif 
    return res;
}





inline void* task_dag_graph::get_true_addr(void* addr, int type)
{
    #ifdef TEMP_LIGHT_BUILD_GRAPH
    void* true_addr = NULL;
    if(type == NORMAL)
    {
        true_addr = addr;
    } 
    else if(type == SHADE) 
    {
        true_addr = get_shade_addr(addr);
    }
    else if(type == UNSHADE)
    {
        true_addr = get_unshade_addr(addr);
    }
    else 
    {
        assert(0);
    }
    return true_addr;
    #else
    return addr;
    #endif
}

#ifdef SPECIFY_END_TASKS
Error_Code task_dag_graph::acemesh_add_end_task(aceMesh_task* t)
{
    end_tasks.insert(t);
}
#endif






#ifdef LOW_COUPLING_GRAPH


void task_dag_graph::add_and_cluster_tuples_to_virtual_task(aceMesh_task* t, void* addr, int type, int significance)
{
    tuple_addr_task  tmp;

    tmp.t = t;
    tmp.addr = addr;
#if !defined(LIGHT_BUILD_GRAPH)
    tmp.type = type;
    tmp.significance = significance;
#endif
    if(type==IN)
        this->r_one_virtual_task_tuples.push_back(tmp);
    else if(type==OUT || type==INOUT)
        this->w_one_virtual_task_tuples.push_back(tmp);
}

inline void task_dag_graph::collect_dep(aceMesh_task* pred_task, aceMesh_task* succ_task,int branch)
{
    this->dep_pair.push_back(std::make_pair(pred_task,succ_task));
    //std::cout<<"branch="<<branch<<",these one dep from "<<pred_task<<" to "<<succ_task<<std::endl;
}

//last of end_split_task
inline void task_dag_graph::reset_virtual_data()
{
    this->w_one_virtual_task_tuples.clear();
    this->r_one_virtual_task_tuples.clear();
    this->dep_pair.clear();
    #ifdef SCAN_NEED_SPAWN
    this->virtual_need_spawn_tasks.clear();
    #endif
}


void task_dag_graph::update_hashmap_and_collect_dep_according_virtual_task_tuple()
{


    for(std::vector<tuple_addr_task>::iterator itr = this->one_virtual_task_tuples.begin(); 
        itr != this->one_virtual_task_tuples.end(); ++itr)
    {
        
        if(itr->type == INOUT_NONE || itr->addr == NULL || itr->t == NULL)
            continue;

        std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr  = this->addr_task.find(itr->addr);

        if(search_itr != addr_task.end())  // if already in hashmap
        {
#ifdef IF_CON_UPDATE
            if(search_itr->second.r_tasks.size()==0 && search_itr->second.w_task.t!=NULL)
            {
                this->collect_dep(search_itr->second.w_task.t,itr->t,1);   //r空时，构造ww边
                if(itr->type == OUT || itr->type == INOUT)
                {
                    search_itr->second.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                    search_itr->second.w_task.significance = itr->significance;
#endif
                }
                else if(itr->type == IN)
                {
                    task_significance t_s;
                    t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                    t_s.significance = itr->significance;
#endif  
                    search_itr->second.r_tasks.push_back(t_s);
                }
            }
            else if(search_itr->second.r_tasks.size()>0)
            {
                if(itr->type == OUT || itr->type == INOUT)
                {
                    for(std::vector<task_significance>::iterator r_task_itr = search_itr->second.r_tasks.begin();  //why without WAW??？
                        r_task_itr != search_itr->second.r_tasks.end(); ++r_task_itr)
                    {                  
                        this->collect_dep(r_task_itr->t,itr->t,2);
                    }

                    search_itr->second.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                    search_itr->second.w_task.significance = itr->significance;
#endif
                    search_itr->second.r_tasks.clear();
                }
                else if(itr->type == IN)
                {
                    if(search_itr->second.w_task.t!=NULL)
                        this->collect_dep(search_itr->second.w_task.t,itr->t,3);
                    task_significance t_s;
                    t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                    t_s.significance = itr->significance;
#endif  
                    search_itr->second.r_tasks.push_back(t_s);
                }
            }
#else

            switch (itr->type) {
                case IN:
                {
                    if (search_itr->second.w_task.t!=NULL) {
//W-R
                        this->collect_dep(search_itr->second.w_task.t,itr->t,1);   //W空时，构造wR边
                    }
//push itself
                    task_significance t_s;
                    t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                    t_s.significance = itr->significance;
#endif  
                    search_itr->second.r_tasks.push_back(t_s);
                }
                    break;
                case OUT:
                case INOUT:
                {
                    if(search_itr->second.r_tasks.size()!=0) {
//iterate build R-W, clear r_tasks
                        for(std::vector<task_significance>::iterator r_task_itr = search_itr->second.r_tasks.begin();  //why without WAW??？
                            r_task_itr != search_itr->second.r_tasks.end(); ++r_task_itr)
                        {                  
                            this->collect_dep(r_task_itr->t,itr->t,2);
                        }
                        search_itr->second.r_tasks.clear();
                    }
                    else {
//W-W
                        this->collect_dep(search_itr->second.w_task.t,itr->t,3);
                    }
//update w task
                    search_itr->second.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                    search_itr->second.w_task.significance = itr->significance;
#endif
                }
                    break;
                default:
                    break;
            }


#endif
        }
        else //if(search_itr == addr_task.end())  // if not in hashmap yet
        {
            rw_task_tuple tmp;  
            if( itr->type == IN )
            {
                task_significance t_s;
                t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                t_s.significance = itr->significance;
#endif
                tmp.r_tasks.push_back(t_s);
            }
            else                        // if write
            {
                tmp.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                tmp.w_task.significance = itr->significance;
#endif
#ifdef SCAN_NEED_SPAWN
                virtual_need_spawn_tasks.push_back(itr->t);
#endif
            }
            this->addr_task[itr->addr] = tmp; 
        }
    }
}
void task_dag_graph::update_hashmap_and_collect_dep_according_virtual_task_tuple_with_clustering()
{
    for(std::vector<tuple_addr_task>::iterator itr = this->w_one_virtual_task_tuples.begin(); 
        itr != this->w_one_virtual_task_tuples.end(); ++itr)
    {
        if(itr->type == INOUT_NONE || itr->addr == NULL || itr->t == NULL)
            continue;
    
        std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr  = this->addr_task.find(itr->addr);

        if(search_itr != addr_task.end())
        {
            if(search_itr->second.r_tasks.size()>0)
            {
                for(std::vector<task_significance>::iterator r_task_itr = search_itr->second.r_tasks.begin();  //why without WAW??？
                    r_task_itr != search_itr->second.r_tasks.end(); ++r_task_itr)
                {                  
                        this->collect_dep(r_task_itr->t,itr->t,4);
                } 
                search_itr->second.r_tasks.clear();
            }
            else
            {
                //if(search_itr->second.r_tasks.size()>0 || search_itr->second.w_task.t ==NULL) assert(0);
                this->collect_dep(search_itr->second.w_task.t,itr->t,5); //w-W

            }
            search_itr->second.w_task.t = itr->t;
        }
        else
        {
            //if(itr->type == OUT || itr->type == INOUT)
            //{
                rw_task_tuple tmp;
                tmp.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                tmp.w_task.significance = itr->significance;
#endif
                this->addr_task[itr->addr] = tmp;
#ifdef SCAN_NEED_SPAWN
                virtual_need_spawn_tasks.push_back(itr->t);
#endif
            //}
             
        }
    }

    for(std::vector<tuple_addr_task>::iterator itr = this->r_one_virtual_task_tuples.begin(); 
        itr != this->r_one_virtual_task_tuples.end(); ++itr)
    {
        if(itr->type == INOUT_NONE || itr->addr == NULL || itr->t == NULL)
            continue;
    
        std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr  = this->addr_task.find(itr->addr);

        if(search_itr != addr_task.end())
        {
            task_significance t_s;
            t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
            t_s.significance = itr->significance;
#endif
            search_itr->second.r_tasks.push_back(t_s); //insert r task whether edges are built or not

            if(search_itr->second.w_task.t!=NULL)
                //this->dep_pair.push_back(std::make_pair(search_itr->second.w_task.t,itr->t)); //W-R
                this->collect_dep(search_itr->second.w_task.t,itr->t,6);
        }
        else
        {
            rw_task_tuple tmp;
            task_significance t_s;
            t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
            t_s.significance = itr->significance;
#endif
            tmp.r_tasks.push_back(t_s);
            this->addr_task[itr->addr] = tmp; 
#ifdef SCAN_NEED_SPAWN
            virtual_need_spawn_tasks.push_back(itr->t);
#endif
        }
    }
}


void task_dag_graph::construct_dep_according_dep_pair()
{
    int ref=0;
    aceMesh_task* local_t=NULL;
    for(std::vector< std::pair<aceMesh_task*,aceMesh_task*> >::iterator itr = this->dep_pair.begin(); 
        itr != this->dep_pair.end(); ++itr)
    {
        //add_successor(itr->first,itr->second);
        ref=add_successor_first_ret(itr->first,itr->second);
        //std::cout<<"ref_count <"<<itr->first<<","<<itr->first->task_base.ref_count_t<<">,<"<<itr->second<<","<<itr->second->task_base.ref_count_t<<">"<<std::endl;


#ifdef TEMP_NEED_SPAWN
        if(local_t!=itr->first)
        {
            if(local_t!=NULL)
            {
                if(local_t->task_base.ref_count_t==-1) 
                    need_spawn_tasks.push_back(itr->first);
            }
            local_t=itr->first;
        }
#endif




#ifdef SCAN_DEP_PAIR
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    unsigned long add_start=rpcc();
#endif
        if(itr->first->task_base.ref_count_t==-1)
        {
            //std::vector<aceMesh_task*>::iterator search_itr  = this->need_spawn_tasks.find(itr->first);
            std::vector<aceMesh_task*>::iterator search_itr  = find(need_spawn_tasks.begin(), need_spawn_tasks.end(),itr->first) ;
            if(search_itr == this->need_spawn_tasks.end())
            {
                need_spawn_tasks.push_back(itr->first);
                //std::cout<<"need_spawn_task:"<<itr->first;
            }
        }
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[NEED_SPAWN_P]+=(rpcc()-add_start);
#endif
#endif
    }
    #ifdef SCAN_NEED_SPAWN
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    unsigned long add_start=rpcc();
#endif
    for(std::vector<aceMesh_task*> ::iterator itr = this->virtual_need_spawn_tasks.begin(); itr != this->virtual_need_spawn_tasks.end(); ++itr)
    {
        if((*itr)->task_base.ref_count_t==-1) 
        {
            std::vector<aceMesh_task*>::iterator search_itr  = find(need_spawn_tasks.begin(), need_spawn_tasks.end(),*itr) ;
            if(search_itr == this->need_spawn_tasks.end())
                need_spawn_tasks.push_back(*itr);
        }
    }
    //std::cout<<"virtual_need_spawn_tasks.size="<<virtual_need_spawn_tasks.size()<<std::endl;
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[NEED_SPAWN_P]+=(rpcc()-add_start);
#endif
    #endif
    //std::cout <<std::endl;
    //std::cout<<"construct_dep_according_dep_pair : need_spawn_tasks: "<<need_spawn_tasks.size()<<std::endl;
    //std::cout<<"construct_dep_according_dep_pair : virtual_need_spawn_tasks: "<<virtual_need_spawn_tasks.size()<<std::endl;
}




#endif

void init_queue_args()
{
    //before use circular_queue, initialize top and bottom
#if defined( _CIRCULAR_QUEUE)
   for(int i=0;i<64;i++)
   {
      schedulers[i].private_queue.bottom=0;
      schedulers[i].private_queue.top=0;
    }
    //ALLERT: bugs in sw5CC, if you use need_spawn_tasks.size(), you will have nightmare. losing the vector totally
    //std::cout<<"begin spawn first level tasks,totally !"<<std::endl;
#else
    alog_debug("====super taskptr:%x", cur_super_taskptr);
    if (cur_super_taskptr == NULL) {
    #ifdef DISTRIBUTED_SCHEDULER
        for(int i=0; i<65; i++) 
    #else
        for(int i=0; i<2; i++) 
    #endif
        {    
#ifdef LOCAL_MULTI_PRIORI
            for(int j=0;j<LOCAL_MULTI_SIZE;j++)
            schedulers[i].private_queue[j].task_pool=NULL;
#else
    #ifdef CENTRAL_FIFO
            schedulers[i].private_queue.head_ptr = NULL;
            schedulers[i].private_queue.tail_ptr = NULL;
    #else
            schedulers[i].private_queue.task_pool=NULL;
      #if defined( ASYN_INSERT_Q ) 
            schedulers[i].lower_private_queue.task_pool=NULL;
      #endif
    #endif
#endif
        }   
    }
#endif

#ifdef _LDMQ
    for(int i=0;i<64;i++)
    {
  #ifdef TARG_SW5
      h2ldm(mqueue_empty,i,cgid)=false;
  #elif TARG_SW9
//      unsigned long mqueue_empty_addr[64];
      my_h2ldm(mqueue_empty_addr[i],i,cgid,int)=false;
  #endif

    }
#endif

//before use mpiqueue, initialize head and tail
#ifdef MASTER
#ifdef MPI_SURPPORT
    if (cur_super_taskptr == NULL) {
        #ifdef DISTRIBUTED_SCHEDULER
        schedulers[total_num_threads].blocking_queue.tail=NULL;
        schedulers[total_num_threads].blocking_queue.head=NULL;//schedulers[sche_num_threads].blocking_queue.tail;
        #else
        schedulers[MS_SCHED].blocking_queue.tail=NULL;
        schedulers[MS_SCHED].blocking_queue.head=NULL;
        #endif
    }
#ifdef DEBUG
    printf("rank:%d,blocking_queue init is over \n",my_mpi_rank);
#endif
#endif

#endif
}

Error_Code task_dag_graph::register_task(aceMesh_task* t, std::vector<addr_tuple >& addrs)
{
//printf("register\n");
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    unsigned long add_start=rpcc();
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) &&  (defined(FINE_PROF2) || defined(FINE_PROF3) )
    unsigned long find_start;
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
   
//printf("concurrent_construct_test\n");
    /*
   t->task_base.state=0;
   t->task_base.edge=0;
   my_mutex_unlock(&(t->task_base.finished_lock));
   my_mutex_unlock(&(t->task_base.successor_lock));
   my_mutex_unlock(&(t->task_base.ref_lock));
   */
  /*lock initial*/
 // set_task_state(0, &(t->task_base.over));

  my_mutex_unlock(&(t->task_base.over));
  my_mutex_unlock(&(t->task_base.successor_lock));
 //t->task_base.over=0;
 //t->task_base.successor_lock=0;
  
  //my_mutex_unlock(&(t->task_base.ref_lock));
  //atomic_sub(val, &(t->task_base.ref_count_t));
  decrement_ref_count(&(t->task_base));
  
#endif
    vector_tasks.push_back(t);  //for destroy

#if (!defined(SPECIFY_END_TASKS)) || (!defined(NO_END_TASK_LIST))
    add_end_task(t);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[END_TASKS_L]+=(rpcc()-add_start);
#endif


#ifdef AUTO_AFFINITY
    if(is_master)
    {
        #ifdef DISTRIBUTED_SCHEDULER
        set_affinity_id(&(t->task_base), total_num_threads);	//modify TODO
        #else
        set_affinity_id(&(t->task_base), MS_SCHED);
        #endif
        is_master=0;
#ifdef DEBUG
        printf("register master affinity:%d\n",t->task_base.affinity_id);
#endif

    }
    else
        set_affinity_id(&(t->task_base), my_affinity.get_affinity_id());
 // printf("register affinity:%d\n",t->task_base.affinity_id);
#ifdef DEBUG
#ifdef SEG_BUFF
#ifndef AGENT_COMPU
    if(t->task_base.affinity_id==total_num_threads-1)
    {
        printf("task_affinity error:%d\n",t->task_base.affinity_id);
        fflush(stdout);
        assert(0);
    }
#endif
#endif
#endif
#ifdef TEMP_AFFI
    t->task_base.bak_affinity_id=t->task_base.affinity_id;
#endif
#endif
#ifdef DEBUG_GRAPH
    task_nums++;
    graph_check.set_task_loop_id(t);
#endif



#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    add_start=rpcc();
#endif


#ifdef LOW_COUPLING_GRAPH
    //std::cout<<"register_task"<<t<<std::endl;
    for(std::vector<addr_tuple>::iterator itr = addrs.begin(); itr != addrs.end(); ++itr)
    {
        if (itr->addr != NULL)
        {
            //get true inout addr
            void* true_addr = this->get_true_addr(itr->addr, itr->area_type);
            //record this inout addr 
#ifdef CLUSTERING
            this->add_and_cluster_tuples_to_virtual_task(t , true_addr, itr->type, 0); 
#else
            this->add_tuples_to_virtual_task(t , true_addr, itr->type, 0);
#endif
            
        }
        if( itr->neighbor == NEIGHBOR_NONE )
            continue;
        //this->neighbor_addrs=NULL;
        num_neighbs=0;
        get_neighbor((ci_task*)t,itr->neighbor, itr->addr, neighbor_addrs,&num_neighbs);
        for(int i=0;i<num_neighbs;++i)
        {
           //get true addr
             void* neighbor_true_addr = this->get_true_addr(neighbor_addrs[i], itr->area_type);
            //record,later read/write to hash_map
#ifdef CLUSTERING
            this->add_and_cluster_tuples_to_virtual_task(t , neighbor_true_addr, itr->neighbor_type, 1);
#else
            this->add_tuples_to_virtual_task(t , neighbor_true_addr, itr->neighbor_type, 1);
#endif
             
        }
    }



#else
  

//printf("next_register\n");
    int sum_pre = 0;
    for(std::vector<addr_tuple>::iterator itr = addrs.begin(); itr != addrs.end(); ++itr)
    {
        if (itr->addr != NULL)
        {
            //get true inout addr
            void* true_addr = this->get_true_addr(itr->addr, itr->area_type);
#ifndef IMMEDIA_UPDATE
            //record this inout addr 
            this->add_tuples_to_virtual_task(t , true_addr, itr->type, 0);
#endif
            #ifdef SAVE_RW_INFO
            add_addr( t, itr->addr, itr->area_type, itr->type, 0);
            #endif        
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
    find_start=rpcc();
#endif
            std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr = addr_task.find(true_addr);
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
            find_hash_ntimes1++;
            buil_prof[FIND_HASH]+=(rpcc()-find_start);
#endif
            if(search_itr != addr_task.end())
            { 
            // #if defined(TARG_SW9)&&defined(CONCURRENT_CONSTRUCT_GRAPH)
            //     con_dest_ptr=(uint64_t)t;
            //     con_type=itr->type;
            //     con_src_ptr=(uint64_t)(search_itr->second);
            //     //con_is_nei=false;
            //     con_status=1;
            //     while(con_status==1);
            //     sum_pre +=con_sum_pre;
            // #else     
                sum_pre += build_releationship(t, itr->type, search_itr->second, false);
            // #endif
            }
#ifdef  IMMEDIA_UPDATE
            else
            {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF3)
                find_start=rpcc();
#endif
                rw_task_tuple tmp;
                if ( itr->type == IN )
                {
                    task_significance t_s;
                    t_s.t = t;
#if !defined(LIGHT_BUILD_GRAPH)
                    t_s.significance = 0;
#endif
                    tmp.r_tasks.push_back(t_s);
                }
                else //if(itr->type == OUT || itr->type == INOUT)
                {
                    tmp.w_task.t = t;
#if !defined(LIGHT_BUILD_GRAPH)
                    tmp.w_task.significance = 0;
#endif  
                }
                this->addr_task[itr->addr] = tmp;
                //this->addr_task.emplace(std::pair<void*, tuple_rw_task>(itr->addr,tmp) ); //has no member named 'emplace'
                //this->addr_task.insert(std::pair<void*, tuple_rw_task>(itr->addr,tmp) );
                //auto result = this->addr_task.insert(std::pair<void*, tuple_rw_task>(itr->addr,tmp) );
                //if (!result.second) {
                ////std::cout << "Key 1 inserted successfully" << std::endl;
                ////} else {
                ///    std::cout << "Key 1 already exists" << std::endl;
                //}
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF3)
                buil_prof[INSERT_HASH]+=(rpcc()-find_start);
#endif
            }
#endif
        }
        if( itr->neighbor == NEIGHBOR_NONE )
            continue;
        //this->neighbor_addrs=NULL;
        num_neighbs=0;
        get_neighbor((ci_task*)t,itr->neighbor, itr->addr, neighbor_addrs,&num_neighbs);
        for(int i=0;i<num_neighbs;++i)
        {
           //get true addr
             void* neighbor_true_addr = this->get_true_addr(neighbor_addrs[i], itr->area_type);
#ifndef IMMEDIA_UPDATE
            //record,later read/write to hash_map
             this->add_tuples_to_virtual_task(t , neighbor_true_addr, itr->neighbor_type, 1);
#endif
             #ifdef SAVE_RW_INFO
             add_addr(t, neighbor_addrs[i], itr->area_type, itr->neighbor_type, 1);
             #endif        
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
    find_start=rpcc();
#endif
             std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr = 
             addr_task.find(neighbor_true_addr);
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
    find_hash_ntimes1++;
    buil_prof[FIND_HASH]+=(rpcc()-find_start);
#endif
             if(search_itr == addr_task.end())
             { 
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF3)
                find_start=rpcc();
#endif
#ifdef IMMEDIA_UPDATE
                rw_task_tuple tmp;
                if ( itr->type == IN )
                {
                    task_significance t_s;
                    t_s.t = t;
#if !defined(LIGHT_BUILD_GRAPH)
                    t_s.significance = 1;
#endif
                    tmp.r_tasks.push_back(t_s);
                }
                else //if(itr->type == OUT || itr->type == INOUT)
                {
                    tmp.w_task.t = t;
#if !defined(LIGHT_BUILD_GRAPH)
                    tmp.w_task.significance = 1;
#endif  
                }
                this->addr_task[itr->addr] = tmp;
                //this->addr_task.emplace(std::pair<void*, tuple_rw_task>(itr->addr,tmp) ); //has no member named 'emplace'
                //this->addr_task.insert(std::pair<void*, tuple_rw_task>(itr->addr,tmp) );
                //auto result = this->addr_task.insert(std::pair<void*, tuple_rw_task>(itr->addr,tmp) );
                //if (!result.second) {
                ////std::cout << "Key 1 inserted successfully" << std::endl;
                ////} else {
                ///    std::cout << "Key 1 already exists" << std::endl;
                //}
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF3)
                buil_prof[INSERT_HASH]+=(rpcc()-find_start);
#endif
                continue;
             }
            // #if defined(TARG_SW9)&&defined(CONCURRENT_CONSTRUCT_GRAPH)
            //     con_dest_ptr=(uint64_t)t;
            //     con_type=itr->neighbor_type;
            //     con_src_ptr=(uint64_t)(search_itr->second);
            //     //con_is_nei=false;
            //     con_status=2;
            //     while(con_status==2);
            //     sum_pre +=con_sum_pre;
            // #else
             sum_pre += build_releationship(t, itr->neighbor_type, search_itr->second, true);
            // #endif
        }

    }

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
    find_start=rpcc();
#endif
#ifdef UNBLOCK_AGENT
//为判断task的前驱地址，加上‘0’进行分隔
     int temp_tail = my_h2ldm(main_tail, total_num_threads - 1, cgid, int);
     while ((temp_tail + 1) % BLOCK_SIZE == my_h2ldm(main_head, total_num_threads - 1, cgid, int))
         ;
     // printf("main_task:%x\n",&(t->task_base));
     my_h2ldm(main_task[temp_tail], total_num_threads - 1, cgid, unsigned long) =
         0;
     my_h2ldm(main_tail, total_num_threads - 1, cgid, int) = (temp_tail + 1) % BLOCK_SIZE;
    asm volatile("memb\n");
//task传给agent，此时前驱地址已经就绪
    // my_h2ldm(con_src_ptr, total_num_threads - 1, cgid, unsigned long) =(unsigned long)t;
    // while(my_h2ldm(con_src_ptr, total_num_threads - 1, cgid, unsigned long));
    // asm volatile("memb\n");
 temp_tail=my_h2ldm(dest_tail,total_num_threads-1,cgid,int);
  while((temp_tail+1)%MULTI_TASK==my_h2ldm(dest_head,total_num_threads-1,cgid,int));
  // printf("dest_task:%x\n",&(t->task_base));
my_h2ldm(dest_task[temp_tail],total_num_threads-1,cgid,unsigned long)=(unsigned long)t;
my_h2ldm(dest_tail,total_num_threads-1,cgid,int)=(temp_tail+1)%MULTI_TASK;
 asm volatile("memb\n");
#else
#ifdef CONCURRENT_CONSTRUCT_GRAPH
    /*
    set_pre(&(t->task_base),sum_pre);
    atomic_add(val,&(t->task_base.state));
    */
#ifdef TARG_SW5
    atomic_add(val,&(t->task_base.ref_count_t));
    if (val == -2)
#else
   // printf("before ref_count:%d\n",t->task_base.ref_count_t);
  #ifndef BLOCK_AGENT
  if(init_queue==0){
  init_queue=1;
  init_queue_args();
  }
  int temp_tail=my_h2ldm(main_tail,total_num_threads-1,cgid,int);
  block_temp=rpcc();
  while((temp_tail+1)%BLOCK_SIZE==my_h2ldm(main_head,total_num_threads-1,cgid,int));
   block_time+=rpcc()-block_temp;
   //printf("main_task:%x\n",&(t->task_base));
   my_h2ldm(main_task[temp_tail],total_num_threads-1,cgid,unsigned long)=(unsigned long)t;
   my_h2ldm(main_tail,total_num_threads-1,cgid,int)=(temp_tail+1)%BLOCK_SIZE;
   asm volatile("memb\n");
   #else
    my_atomic_add(&(t->task_base.ref_count_t));
    val=con_sum_pre;
   #endif
   // printf("after val:%d,ref_count:%d\n",val,t->task_base.ref_count_t);
    if(val==-1)
#endif
    
#else
    //printf("sum_pre:%d ref_count:%d\n",sum_pre,t->task_base.ref_count_t);
    if (sum_pre == 0)  // if no any dependency edges are added in func(build_rela)
#endif
    { 
      //  printf("sum_pre==0\n");
      #if defined(BLOCK_AGENT)|| !defined(CONCURRENT_CONSTRUCT_GRAPH)
      need_spawn_tasks.push_back(t);
      #endif
#ifdef AUTO_PARTITION
        my_disjoin_set.make_set(sep_id);
        sep_task[sep_id] = t;
        t->set_group_id(sep_id++);
#endif

#ifdef SUPPORT_PARTITION
        int id = t->get_part_id();
        std::map<int, std::vector<aceMesh_task*> >::iterator itr = sep_data.find( id );
        if(itr == sep_data.end() )
        {
            std::vector<aceMesh_task*> tmp;
            tmp.push_back(t);
            sep_data.insert( std::pair<int,std::vector<aceMesh_task*> >(id, tmp) );
        }
        else 
        {
            itr->second.push_back(t);
        }
#endif
    }
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
    buil_prof[NEED_SPAWN_P]+=(rpcc()-find_start);
#endif
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[BUILD_RELAT]+=(rpcc()-add_start);
#endif
    return ACEMESH_OK;
}

void task_dag_graph::add_tuples_to_virtual_task(aceMesh_task* t, void* addr, int type, int significance)
{
    tuple_addr_task  tmp;
    tmp.t = t;
    tmp.type = type;
    tmp.addr = addr;
#if !defined(LIGHT_BUILD_GRAPH)
    tmp.significance = significance;
#endif
    this->one_virtual_task_tuples.push_back(tmp);
}

/*#ifdef USE_PRIORITY_QUEUE
bool compare_task2(aceMesh_task* t1, aceMesh_task* t2)
{ 
    //large than...., same as compare_task in class generic_scheduler
    return t1->get_priority_id() > t2->get_priority_id();
}
#endif
*/
/*
void init_queue_args()
{
    //before use circular_queue, initialize top and bottom
#if defined( _CIRCULAR_QUEUE)
   for(int i=0;i<64;i++)
   {
      schedulers[i].private_queue.bottom=0;
      schedulers[i].private_queue.top=0;
    }
    //ALLERT: bugs in sw5CC, if you use need_spawn_tasks.size(), you will have nightmare. losing the vector totally
    //std::cout<<"begin spawn first level tasks,totally !"<<std::endl;
#else
    alog_debug("====super taskptr:%x", cur_super_taskptr);
    if (cur_super_taskptr == NULL) {
    #ifdef DISTRIBUTED_SCHEDULER
        for(int i=0; i<65; i++) 
    #else
        for(int i=0; i<2; i++) 
    #endif
        {    
#ifdef LOCAL_MULTI_PRIORI
            for(int j=0;j<LOCAL_MULTI_SIZE;j++)
            schedulers[i].private_queue[j].task_pool=NULL;
#else
    #ifdef CENTRAL_FIFO
            schedulers[i].private_queue.head_ptr = NULL;
            schedulers[i].private_queue.tail_ptr = NULL;
    #else
            schedulers[i].private_queue.task_pool=NULL;
      #if defined( ASYN_INSERT_Q ) 
            schedulers[i].lower_private_queue.task_pool=NULL;
      #endif
    #endif
#endif
        }   
    }
#endif

#ifdef _LDMQ
    for(int i=0;i<64;i++)
    {
  #ifdef TARG_SW5
      h2ldm(mqueue_empty,i,cgid)=false;
  #elif TARG_SW9
//      unsigned long mqueue_empty_addr[64];
      my_h2ldm(mqueue_empty_addr[i],i,cgid,int)=false;
  #endif

    }
#endif

//before use mpiqueue, initialize head and tail
#ifdef MASTER
#ifdef MPI_SURPPORT
    if (cur_super_taskptr == NULL) {
        #ifdef DISTRIBUTED_SCHEDULER
        schedulers[total_num_threads].blocking_queue.tail=NULL;
        schedulers[total_num_threads].blocking_queue.head=NULL;//schedulers[sche_num_threads].blocking_queue.tail;
        #else
        schedulers[MS_SCHED].blocking_queue.tail=NULL;
        schedulers[MS_SCHED].blocking_queue.head=NULL;
        #endif
    }
#ifdef DEBUG
    printf("rank:%d,blocking_queue init is over \n",my_mpi_rank);
#endif
#endif

#endif
}
*/
Error_Code task_dag_graph::spawn()
{
/*#ifdef USE_PRIORITY_QUEUE
    if(0){
           //it is not necessary to sort the list before entering a red-black tree
          std::sort(need_spawn_tasks.begin(), need_spawn_tasks.end(), compare_task2);
    }
#endif
*/
#ifdef _EXEC_TIME
    //gettimeofday(&start,NULL); 
    if(first_graph_spawn){
  #ifdef TARG_SW5
        start_count=rpcc();
  #elif TARG_SW9
        start_count=athread_time_cycle();
  #endif
        first_graph_spawn=0;
    }
#endif


#ifndef CONCURRENT_CONSTRUCT_GRAPH
#ifdef DEBUG_GRAPH
    decrement_ref_count(&(endTask->aceMesh_task_base.task_base)); 
#else
    decrement_ref_count(&(endTask->task_base));
#endif
#endif


#ifndef CONCURRENT_CONSTRUCT_GRAPH
#if defined( _CIRCULAR_QUEUE)
   for(int i=0;i<64;i++)
   {
      schedulers[i].private_queue.bottom=0;
      schedulers[i].private_queue.top=0;
    }
#else
    alog_debug("====super taskptr:%x", cur_super_taskptr);
    if (cur_super_taskptr == NULL) {
    #ifdef DISTRIBUTED_SCHEDULER
        for(int i=0; i<65; i++) 
    #else
        for(int i=0; i<2; i++) 
    #endif
        {    
#ifdef LOCAL_MULTI_PRIORI
            for(int j=0;j<LOCAL_MULTI_SIZE;j++)
            schedulers[i].private_queue[j].task_pool=NULL;
#else
    #ifdef CENTRAL_FIFO
            schedulers[i].private_queue.head_ptr = NULL;
            schedulers[i].private_queue.tail_ptr = NULL;
    #else
            schedulers[i].private_queue.task_pool=NULL;
      #if defined( ASYN_INSERT_Q ) 
            schedulers[i].lower_private_queue.task_pool=NULL;
      #endif
    #endif
#endif
        }   
    }
#endif

#endif
//struct task temp[3];
int task_num=0,iter=0;
   // my_h2ldm(con_status,63,cgid,int)=3;
   //  asm volatile("memb\n");
    for(std::vector<aceMesh_task*>::iterator itr = need_spawn_tasks.begin(); itr != need_spawn_tasks.end(); ++itr)
    {
        alog_debug("graph: %x, spawning: %x", this, &(*itr)->task_base);
    #if defined(SUCC_BATCH) && defined(CONCURRENT_CONSTRUCT_GRAPH) && (_SERIAL_QUEUE)
        int destid = get_affinity_id(&(*itr)->task_base);
       // printf("spawn\n");
        if(M2S_detail_temp_64[destid].task_num<PENDING_TASK_NUM){
            M2S_detail_temp_64[destid].task_detail[M2S_detail_temp_64[destid].task_num]=&(*itr)->task_base;					  					  
            M2S_detail_temp_64[destid].task_num++;
        }else{
            printf("m2s_temp[%d] is full\n",destid);
            assert(0);
        }
    #else
        #if defined(EMP_MASTER) && defined(CONCURRENT_CONSTRUCT_GRAPH)
        if ((*itr)->task_base.affinity_id == total_num_threads) {
            EQ_main_push(&(*itr)->task_base, STATUS_PUSH_MS);
        }
        else {
            if (!master_buff_push(&ready_succ_buff, &(*itr)->task_base)) { //buffer full
                while (m2s_flag != BUFF_STATUS_MS) {
                    ;
                }
                master_buff_consume();
                m2s_flag = BUFF_STATUS_SL;
                asm volatile ("memb\n");
                master_buff_push(&ready_succ_buff, &(*itr)->task_base);
            }
        }
        #else
            #ifdef CONCURRENT_CONSTRUCT_GRAPH
            #if defined(TARG_SW9)// && !defined(EMP_QUEUE) 
            #ifdef MULTI_SPAWN
     //      while(my_h2ldm(spawn_status[iter],63,cgid,int));
           //printf("spawn\n");
           if(task_num<MULTI_TASK){       
            //my_spawn_to_id(&(*itr)->task_base);
              temp_task[task_num]=(unsigned long)&(*itr)->task_base; 
              task_num++;
            }else{
              my_spawn_to_id(task_num);
             // iter=(iter+1)%MULTI_STATUS;
              task_num=0;
            //  while(my_h2ldm(spawn_status[iter],63,cgid,int));
              temp_task[task_num]=(unsigned long)&(*itr)->task_base;
              task_num++;
            }
            #else
            //temp_task[0]=&(*itr)->task_base;
            my_spawn_to_id(&(*itr)->task_base);
            #endif
            #else
            //printf("spawn\n");
            spawn_to_id(&(*itr)->task_base);
            #endif
            #else
        if (cur_super_taskptr != NULL){
            spawn_to_id(&(*itr)->task_base);
        }else{
        //printf("init_spawn\n");
         init_spawn(&(*itr)->task_base);
        // my_spawn_to_id(&(*itr)->task_base);
         }
         #endif
        #endif
    #endif
    }
    #if defined(CONCURRENT_CONSTRUCT_GRAPH) && defined(TARG_SW9)&& defined(MULTI_SPAWN)
     if(task_num>0)
     {my_spawn_to_id(task_num);
//     iter=(iter+1)%MULTI_STATUS;
  //    while(my_h2ldm(spawn_status[iter],63,cgid,int));
   //   my_h2ldm(spawn_status[iter],63,cgid,int)=-1;
    //  asm volatile("memb\n");
     }
     /*
     else{
     my_h2ldm(spawn_status[iter],63,cgid,int)=-1;
      asm volatile("memb\n");
     }
   while(my_h2ldm(con_status,63,cgid,int));
    */
    #endif
    //init_queue_args();
#ifndef CONCURRENT_CONSTRUCT_GRAPH
    //
    #ifdef _LDMQ
    for(int i=0;i<64;i++)
    {
        #ifdef TARG_SW5
      h2ldm(mqueue_empty,i,cgid)=false;
        #elif TARG_SW9
     // unsigned long mqueue_empty_addr[64];
      my_h2ldm(mqueue_empty_addr[i],i,cgid,int)=false;
        #endif
    }
     #endif
#endif
#if defined(SUCC_BATCH) && defined(CONCURRENT_CONSTRUCT_GRAPH) && (_SERIAL_QUEUE)
    emp_d_master_consum_all();
#endif
#if defined(EMP_MASTER) && defined(CONCURRENT_CONSTRUCT_GRAPH)
    while (m2s_flag != BUFF_STATUS_MS) {
        ;
    }
    master_buff_consume();
    m2s_flag = BUFF_STATUS_SL;
    asm volatile ("memb\n");
#endif
    return ACEMESH_OK;
}


Error_Code task_dag_graph::wait()
{
    wait_for_all((task*)(this->endTask));
    this->destroy_task_graph();
    this->reset_task_graph();
    return ACEMESH_OK;
}

#ifdef AUTO_PARTITION
void task_dag_graph::spawn(int sep_id)
{
    std::vector<int>& my_sep_data = sep_data[sep_id];

    for(std::vector<int>::iterator itr = my_sep_data.begin(); itr != my_sep_data.end(); ++itr)
    {
        task::spawn(*sep_task[*itr]);
    }
}

void task_dag_graph::end_construct_dag_separation()
{
    sep_end_tasks.clear();
    for(std::unordered_set<aceMesh_task*>::iterator itr = this->end_tasks.begin(); 
        itr != this->end_tasks.end(); ++itr)
    {
        int id = my_disjoin_set.find_set( (*itr)->get_group_id() );
        std::map<int, task*>::iterator itr_find = sep_end_tasks.find(id);
        task* tmp = NULL;
        if(itr_find == sep_end_tasks.end())
        {
#ifdef DEBUG_GRAPH
            tmp = &end_task_constructor();
#else
            tmp = &empty_task_constructor();
#endif
            sep_end_tasks.insert(std::pair<int,task*>(id, tmp));
        }
        else 
        {
            tmp = itr_find->second;
        }
        (*itr)->add_end_successor(tmp);
    }
}

//no use
Error_Code task_dag_graph::spawn_and_wait_with_separation()
{
    int sep_num = my_disjoin_set.get_sep_data(sep_data);
    this->end_construct_dag_separation();

#ifdef _EXEC_TIME
   gettimeofday(&start,NULL); 
#endif

    int i = 0;
    for(std::map<int, std::vector<int> >::iterator map_itr = sep_data.begin(); map_itr != sep_data.end(); ++map_itr)
    {
        std::vector<int>& my_sep_data = map_itr->second;
        int id = map_itr->first;
        task* my_part = sep_end_tasks[id];
        my_part->increment_ref_count();
        for(std::vector<int>::iterator itr = my_sep_data.begin(); itr != my_sep_data.end(); ++itr)
        {
            task::spawn(*sep_task[*itr]);
        }
#ifdef ACEMESH_PERFORMANCE
        my_perf.record_start();
#endif
        my_part->wait_for_all();
#ifdef ACEMESH_PERFORMANCE
        my_perf.record_end();
#endif
    }

#ifdef _EXEC_TIME
    gettimeofday(&end,NULL);
    exec_time+=(end.tv_sec-start.tv_sec)+(end.tv_usec-start.tv_usec)/1000000.0;
#endif

    this->destroy_task_graph();
    this->reset_task_graph();
    sep_id = 0;
    my_disjoin_set.clear();
    return ACEMESH_OK;
}
#endif

#ifdef DEBUG_GRAPH
Error_Code task_dag_graph::spawn_and_wait(int print_graph)
{
    this->g_print_graph = true;
    return this->spawn_and_wait();
}
#endif

#ifdef REUSE_GRAPH
void task_dag_graph::_store_dag_info()
{
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF) 
    unsigned long restore_start=rpcc();
#endif
    aceMesh_task *p_task = NULL;
    //std::cout << "len of vector_tasks: " << vector_tasks.size() << std::endl;
    for(std::vector<aceMesh_task*>::iterator itr = vector_tasks.begin(); itr != vector_tasks.end(); ++itr)
    {
        p_task= *itr;
        p_task->task_base.backup_ref = p_task->task_base.ref_count_t;
        p_task->task_base.reused = true;
    }
    this->endTask->task_base.reused = true;
    this->endTask->task_base.backup_ref = this->endTask->task_base.ref_count_t;
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[RESTROE_GRAPH]+=(rpcc()-restore_start);
#endif
}

void task_dag_graph::free_tasks()
{
    //endTask not instance of aceMesh_task, can not cast it
    for (std::tr1::unordered_map<int, dag_instance>::iterator mapitr = dag_instance_map.begin(); mapitr != dag_instance_map.end(); ++mapitr) {
        need_spawn_tasks = mapitr->second.need_spawn_tasks;
        wait_free_tasks.clear();
        for (std::vector<aceMesh_task *>::iterator itr = need_spawn_tasks.begin(); itr != need_spawn_tasks.end(); ++itr)
        {
            //(*itr)->free_all(wait_free_tasks);
        }
        for (std::tr1::unordered_set<task *>::iterator itr = wait_free_tasks.begin(); itr != wait_free_tasks.end(); ++itr)
        {
            delete (*itr);
        }
    }
}

void task_dag_graph::_save_to_buffer()
{
    //std::cout << "save point 0 reach, dagNo: " << curr_dagNo  << std::endl;
    dag_instance tmp_instance = {
        this->endTask,
    #ifdef CONCURRENT_CONSTRUCT_GRAPH
        this->init_spawn_tasks,
    #else
        this->need_spawn_tasks,
    #endif
        this->vector_tasks
    };
    dag_instance_map[curr_dagNo] = tmp_instance;
    //std::cout << "save point 1 reach, sizeof nst: " << this->need_spawn_tasks.size()  << std::endl;
}

void task_dag_graph::_resume_from_buffer()
{
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF) 
    unsigned long restore_start=rpcc();
#endif
    dag_instance tmp_instance = dag_instance_map[curr_dagNo]; 
    this->endTask = tmp_instance.endTask;
    //std::cout << "_reuse_from  ref_count_t: " << this->endTask->task_base.ref_count_t << std::endl;
    this->need_spawn_tasks = tmp_instance.need_spawn_tasks;
    //std::cout << "_reuse_from len need_sp: " << this->need_spawn_tasks.size() << std::endl;
    this->vector_tasks = tmp_instance.vector_tasks;
    //std::cout << "_reuse_from len vector_tasks: " << this->vector_tasks.size() << std::endl;
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[RESTROE_GRAPH]+=(rpcc()-restore_start);
#endif
}
#endif


#ifdef ACEMESH_PROFILING_CLASSIFY
Error_Code task_dag_graph::spawn_and_wait(int classify_id)
#else
Error_Code task_dag_graph::spawn_and_wait()
#endif
{
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_spawn_and_wait);
#endif
    int i;
#ifdef CONCURRENT_CONSTRUCT_GRAPH
printf("mainsucc:%ld,mainnum:%d\n",main_succ,main_num);
printf("main_block_time:%ld,time:%lf\n",block_time,block_time/MFREQ);
#endif
#ifdef REUSE_GRAPH
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF) 
    unsigned long restore_start=rpcc();
#endif
    //reuse
    bool reuse = false;
    bool init = false;

    //std::cout << "sizeof dag_reuse_flag: " << dag_reuse_flag.size() << std::endl;
    std::tr1::unordered_map<int, bool>::iterator iter = dag_reuse_flag.find(curr_dagNo);
    //std::cout << "point 1 reach" << std::endl;
    if (iter != dag_reuse_flag.end()) {
       reuse = iter->second;
       //std::cout << "point 2 reach" << std::endl;
       init = dag_reuse_cnt[curr_dagNo] == -1 ? true : false;
       //std::cout << "point 3 reach, reuse: " << reuse << " init: " << init << std::endl;
    }

    if (reuse) {//when dag_start return 1
        //resume dag
        this->_resume_from_buffer();
        graph_reuse_times++;
        //std::cout << "point 4 reach" << std::endl;
    } else {
        this->end_construct_dag();
        if (init) {
            dag_reuse_cnt[curr_dagNo] = 0;

            this->_store_dag_info();
            //save dag to buffer
            //std::cout << "point 5 reach" << std::endl;
            this->_save_to_buffer();
            //std::cout << "point 6 reach" << std::endl;
        }
    }
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[RESTROE_GRAPH]+=(rpcc()-restore_start);
#endif
#else

    this->end_construct_dag();
#endif

    alog_debug("endTask ref: %d", this->endTask->task_base.ref_count_t);
    alog_debug("endTask addr: %x", this->endTask);
    alog_debug("need_spawn_tasks: %d", need_spawn_tasks.size());
    std::cout<<"need_spawn_tasks: "<<need_spawn_tasks.size()<<std::endl;
    fflush(stdout);
#ifdef _DAG_TIME      //start in dag_start()
    dag_end_count=rpcc();
#ifdef REUSE_GRAPH
    if (reuse) {
        dag_reuse_time += (float)(dag_end_count-dag_start_count)/MFREQ;
        alog_info("----dag reuse time: %f", dag_reuse_time);
        //printf("----dag reuse time: %lf\n", dag_reuse_time);
    } else {
#endif
        dag_build_time += (float)(dag_end_count-dag_start_count)/MFREQ;
        alog_info("----dag build time: %f", dag_build_time);
        //printf("----dag build time: %lf\n", dag_build_time);
#ifdef REUSE_GRAPH
    }
#endif
    dag_start_count=0; 
#endif


#ifdef DEBUG
    if (my_mpi_rank <= 0) {
        std::cout<< my_mpi_rank <<"acemesh::end of end_construct_dag\n";
    }
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING  //start in first begin_split_task()
        if(first_graph_build==0){ 
            graph_build_end=rpcc();
            first_graph_build=1;
            std::cout << "cur_task_graph_time = "<<(double)(graph_build_end-graph_build_start)/MFREQ << std::endl;
            buil_prof[T_GRAPH_BUILD]+=(graph_build_end-graph_build_start);
        }
#endif
//----------------------------------------------------------
//--------------taskgraph building is over------------------
//----------------------------------------------------------
/*
#ifdef _EXEC_TIME
    //gettimeofday(&start,NULL); 
    if(first_graph_spawn){
  #ifdef TARG_SW5
        start_count=rpcc();
  #elif TARG_SW9
        start_count=athread_time_cycle();
  #endif
        first_graph_spawn=0;
    }
#endif
*/

#ifdef ACEMESH_PERFORMANCE
    my_perf.record_start();
#endif

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    unsigned long e_start=rpcc();
#endif
    //std::cout << "p2 refcnt: " << this->endTask->task_base.ref_count_t << std::endl;
#if (!defined(CONCURRENT_CONSTRUCT_GRAPH)) ||(defined(CONCURRENT_CONSTRUCT_GRAPH) && defined(FAKE_CONCURRENT_GRAPH))
    this->spawn();
    std::cout<< my_mpi_rank <<"acemesh::end of spawn\n";
#endif

    //std::cout << "p3 refcnt: " << this->endTask->task_base.ref_count_t << std::endl;
//#ifdef DEBUG
//    if (my_mpi_rank <= 0) {
        
        //is_run = 10;
        //mpi_is_run = 10;
//        std::cout << "cgid=" << cgid <<"my_mpi_id="<<my_mpi_rank<< std::endl;
//    }
//#endif
#ifdef LOCAL_FLAG
#ifdef _SPAWN
    for(i=0;i<64;i++)
#else
    for(i=0;i<total_num_threads;i++)
#endif
    {
#ifdef TARG_SW5
#ifdef _CIRCULAR_QUEUE
        h2ldm(init_flag,i,cgid)=1;
#endif
        h2ldm(is_run,i,cgid)=1;
#elif TARG_SW9
        my_h2ldm(is_run_addr[i],i,cgid,int)=1;
#ifdef DEBUG
        printf("is_run=%d,%d\n",i,my_h2ldm(is_run_addr[i],i,cgid,int));
#endif
#endif
    }
#else
    is_run = 1;
    //printf("start working\n");

#endif
#ifdef EMP_QUEUE
while(eq_status!=STATUS_IDLE);
my_h2ldm(master_spawn,0,cgid,int)=1;
#endif
//std::cout << "cgid=" << cgid <<"my_mpi_id="<<my_mpi_rank<< std::endl;
// #ifndef SPEIO
// con_graph=0;
//  asm volatile("memb\n");
// #endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    //unsigned long e_end=rpcc();
    buil_prof[CLEAR_NEED_SPAWN]+=rpcc()-e_start;
#endif

#ifdef ACEMESH_PARA
    exec_from=rpcc();
    if(first)
    {
        time_init=rpcc();
        exec_from=time_init;
        exec_to=time_init;
        first=0;
#ifdef DEBUG
        printf("first_time_init:%ld\n",time_init);
#endif
    }
    tt_build+=(exec_from-exec_to);
#endif


    //std::cout << "p4 refcnt: " << this->endTask->task_base.ref_count_t << std::endl;

#ifdef DEBUG 
    std::cout<< my_mpi_rank <<"acemesh::before_wait_for_all\n";
    fflush(stdout);
#endif
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_wait_for_all_task);
#endif
    wait_for_all((task*)(this->endTask));
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
#ifndef SPEIO
   con_graph=0; 
   asm volatile("memb\n");
#endif

#ifdef ACEMESH_PARA
    exec_to=rpcc();
    gid3=(exec_to-time_init-tt_build)/interval;
    //printf("gdi3:%ld,%ld,%ld,%d,%d\n",exec_to,time_init,tt_build,gid3,interval);
    //printf("gid3:%ld,%d,%d,%lf\n",exec_to-time_init-tt_build,interval,
    //        (exec_to-time_init-tt_build)/(long int)interval,
    //        (exec_to-time_init-tt_build)/(long int)interval);
#endif

#ifdef DEBUG
    std::cout<< my_mpi_rank <<"acemesh::end_wait_for_all\n";
    fflush(stdout);
#endif
    //mpi_is_run = 0;
#ifdef ACEMESH_PERFORMANCE
    my_perf.record_end();
#endif

#ifdef _EXEC_TIME
    //gettimeofday(&end,NULL);
    if(!first_graph_spawn){
    #ifdef TARG_SW5
        end_count=rpcc();
    #elif TARG_SW9
        end_count=athread_time_cycle();
    #endif
        exec_count+=(end_count-start_count);
    #ifdef ACEMESH_GRAPH_BUILD_PROFILING
        buil_prof[T_GRAPH_EXEC]=exec_count;
        #ifdef REUSE_GRAPH
            #ifdef USE_COMPOSITE_TASK
        buil_prof[T_GRAPH_TIME]=(end_count-total_start);
            #else
        buil_prof[T_GRAPH_TIME]=(end_count-graph_build_start);
            #endif
        #else
            #ifdef USE_COMPOSITE_TASK
        buil_prof[T_GRAPH_TIME]+=(end_count-total_start);
            #else
        buil_prof[T_GRAPH_TIME]+=(end_count-graph_build_start);
        //buil_prof[T_GRAPH_TIME]=(end_count-graph_build_start);
            #endif
        #endif
    #endif
    #ifdef ACEMESH_SCHEDULER_PROFILING
        if (my_mpi_rank <= 0) { 
      //std::cout << start_count<<","<<end_count<<std::endl;
          std::cout<< "each-spawn-and-wait:" << my_mpi_rank << ", etime : " << (float)(end_count-start_count)/MFREQ << std::endl;
      //std::cout<< "each-spawn-and-wait:" << my_mpi_rank << ", etime : " << (float)(exec_count)/MFREQ << std::endl;
    //exec_time+=(end.tv_sec-start.tv_sec)+(end.tv_usec-start.tv_usec)/1000000.0;    
        }
    #endif
    #ifdef ACEMESH_PROFILING_CLASSIFY
        classify_exec_time[classify_id]+=(float)(end_count-start_count)/MFREQ;
    #endif
        first_graph_spawn=1;
    }
#endif

    //std::cout << "p5 refcnt: " << this->endTask->task_base.ref_count_t << std::endl;
#ifdef LOCAL_FLAG
#ifdef _SPAWN
    for(i=0;i<64;i++)
#else
    for(i=0;i<total_num_threads;i++)
#endif
    {
        if (cur_super_taskptr == NULL) 
        {
#ifdef TARG_SW5
        #ifdef _CIRCULAR_QUEUE
            h2ldm(init_flag,i,cgid)=0;
        #endif
            h2ldm(is_run,i,cgid)=0;
#elif TARG_SW9
            my_h2ldm(is_run_addr[i],i,cgid,int)=0;
#endif
        }
    }
#else
    is_run=0;
#endif
#ifdef DEBUG
    std::cout<< "cgid="<<cgid<<","<<my_mpi_rank <<" acemesh::end_h2ldm_of_wait_for_all\n";
#endif

    //this->destroy_task_graph();
    //std::cout << "p5 refcnt: " << this->endTask->task_base.ref_count_t << std::endl;

#ifdef REUSE_GRAPH
    if (this->endTask->task_base.reused)
    {
        this->endTask->task_base.ref_count_t = this->endTask->task_base.backup_ref;
    }
#endif
    this->reset_task_graph();
    
/*    std::vector<aceMesh_task*>::iterator same_itr;
    std::vector<aceMesh_task*>::iterator copy_same_itr;
    std::vector<aceMesh_task*>::iterator del_same_itr;
    std::vector<aceMesh_task*> del_vec;
    for(std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr = this->addr_task.begin();
         search_itr != this->addr_task.end(); ++search_itr)
    {
        if(search_itr->second.w_task.t != NULL )
        {
           for( same_itr = del_vec.begin(); same_itr != del_vec.end(); ++same_itr)
           {
             if((get_loop_id(*same_itr)) == (get_loop_id(search_itr->second.w_task.t)) &&
                 (get_task_id(*same_itr)) == (get_task_id(search_itr->second.w_task.t)))
                    break;
            }            
           if(same_itr == del_vec.end())
           {
             del_vec.push_back(search_itr->second.w_task.t);
           }
        }
        for(std::vector<task_significance>::iterator r_task_itr = search_itr->second.r_tasks.begin();
              r_task_itr != search_itr->second.r_tasks.end(); ++r_task_itr)
              {
                  for(copy_same_itr = del_vec.begin(); copy_same_itr != del_vec.end(); ++copy_same_itr)
                  {        
                     if((get_loop_id(*copy_same_itr)) == (get_loop_id(r_task_itr->t)) &&
                         (get_task_id(*copy_same_itr)) == (get_task_id(r_task_itr->t)))                                      
                            break;
                   }                       
                   if(copy_same_itr == del_vec.end())              
                      {    
                        del_vec.push_back(r_task_itr->t);
                      }
              }
    }
    for(del_same_itr = del_vec.begin(); del_same_itr != del_vec.end(); ++del_same_itr)
    {
        free(*del_same_itr); 
        *del_same_itr = NULL;                          
    }
    
    
*/    
    
#ifdef AUTO_PARTITION
    sep_id = 0;
    my_disjoin_set.clear();
#endif     

    #ifdef EMP_QUEUE_PROFILING
    printf("EMP Queue MAX Length: %u\n", max_length);
    #endif
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
    return ACEMESH_OK;
}

#ifdef CONCURRENT_CONSTRUCT_GRAPH
Error_Code task_dag_graph::wait_for_all_task()
{
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_wait_for_all_task);
#endif
   this->end_construct_dag();
   int i;
   spawn_num=0;

#ifdef ACEMESH_PERFORMANCE
    my_perf.record_start();
#endif
    //decrement_ref_count(&(endTask->task_base));

    wait_for_all((task*)(this->endTask));
#ifdef ACEMESH_PERFORMANCE
    my_perf.record_end();
#endif

#ifdef LOCAL_FLAG
#ifdef _SPAWN
    for(i=0;i<64;i++)
#else
    for(i=0;i<total_num_threads;i++)
#endif
    {
        if (cur_super_taskptr == NULL) 
        {
#ifdef TARG_SW5
        #ifdef _CIRCULAR_QUEUE
            h2ldm(init_flag,i,cgid)=0;
        #endif
            h2ldm(is_run,i,cgid)=0;
#elif TARG_SW9
            my_h2ldm(is_run_addr[i],i,cgid,int)=0;
#endif
        }
    }
#else
    is_run=0;
#endif
    this->reset_task_graph();
    std::vector<aceMesh_task *>::iterator same_itr;
    std::vector<aceMesh_task *>::iterator copy_same_itr;
    std::vector<aceMesh_task *>::iterator del_same_itr;
    std::vector<aceMesh_task *> del_vec;
    //empty_task* endTask_check;
    //del_vec.push_back(endTask_check);
    for (std::tr1::unordered_map<void *, tuple_rw_task>::iterator search_itr = this->addr_task.begin();
         search_itr != this->addr_task.end(); ++search_itr)
    {
        if (search_itr->second.w_task.t != NULL)
        {
            for (same_itr = del_vec.begin(); same_itr != del_vec.end(); ++same_itr)
            {
                //copy_same_itr = same_itr;
                // std::cout<<"-----------------------first--------------------------------------------------"<<std::endl;
                if ((get_loop_id((*same_itr))) == (get_loop_id(search_itr->second.w_task.t)) &&
                    (get_task_id((*same_itr))) == (get_task_id(search_itr->second.w_task.t)))
                    break;
            }
            if (same_itr == del_vec.end())
            {
                del_vec.push_back(search_itr->second.w_task.t);
                // std::cout<<"del_vec size first:"<<del_vec.size()<<std::endl;
            }
        }
        for (std::vector<task_significance>::iterator r_task_itr = search_itr->second.r_tasks.begin();
             r_task_itr != search_itr->second.r_tasks.end(); ++r_task_itr)
        {
            for (copy_same_itr = del_vec.begin(); copy_same_itr != del_vec.end(); ++copy_same_itr)
            {
                //copy_same_itr = same_itr;
                //std::cout<<"--------------------second-------------------------------------------------"<<std::endl;
                if ((get_loop_id((*copy_same_itr))) == (get_loop_id(r_task_itr->t)) &&
                    (get_task_id((*copy_same_itr))) == (get_task_id(r_task_itr->t)))
                    break;
            }
            if (copy_same_itr == del_vec.end())
            {
                //std::cout<<"same_itr loop id:"<<(*copy_same_itr)->get_loop_id()<<"same_itr task id:"<<(*copy_same_itr)->get_task_id()<<std::endl;
                //std::cout<<"r_task_itr loop id:"<<r_task_itr->t->get_loop_id()<<"r_task_itr task id:"<<r_task_itr->t->get_task_id()<<std::endl;
                del_vec.push_back(r_task_itr->t);
                //    std::cout<<"del_vec size second:"<<del_vec.size()<<std::endl;
            }
        }
    }
    for (del_same_itr = del_vec.begin(); del_same_itr != del_vec.end(); ++del_same_itr)
    {
        //std::cout<<"-----------------------three-------------------------------------------------"<<std::endl;
        //  std::cout<<"del_vec size:"<<del_vec.size()<<std::endl;
        delete *del_same_itr;
        *del_same_itr = NULL;
    }
#ifdef AUTO_PARTITION
    sep_id = 0;
    my_disjoin_set.clear();
#endif
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
    return ACEMESH_OK;
}
#endif

void _move_successor(aceMesh_task *super_task, aceMesh_task *endtask)
{
    //deep copy
    //endtask->task_base = super_task->task_base;
    endtask->capacity_addrs = super_task->capacity_addrs;
    endtask->successor_count = super_task->successor_count;
    //super_task->successor_count = 0;
    endtask->vertical_task = super_task->vertical_task;
    endtask->task_base.ref_count_t++; 
    endtask->task_base.affinity_id = super_task->task_base.affinity_id; 
    
    //std::cout << "ref_count:" << endtask->task_base.ref_count_t << std::endl;
    //std::cout << "successor_count:" << endtask->successor_count << std::endl;
    //std::cout << "type:" << endtask->my_type << std::endl;

    //TODO: support memory pool
#ifdef MEMORY_POOL
 #ifdef USE_SIMD
  #ifdef TARG_SW5
    endtask->successor_tasks = (struct task**)acemesh_myalloc_aligned_32(sizeof(struct task*) * super_task->capacity_addrs);
  #elif TARG_SW9
    endtask->successor_tasks = (struct task**)acemesh_myalloc_aligned_64(sizeof(struct task*) * super_task->capacity_addrs);
  #endif
 #else
    endtask->successor_tasks = (struct task**)acemesh_myalloc_aligned_16(sizeof(struct task*) * super_task->capacity_addrs);
 #endif
#else
    endtask->successor_tasks = (struct task**)malloc(sizeof(struct task*) * super_task->capacity_addrs);
    //endtask->successor_tasks = (struct task**)my_malloc_aligned(sizeof(struct task*) * super_task->capacity_addrs);
#endif
    memcpy(endtask->successor_tasks, super_task->successor_tasks,
           sizeof(struct task *) * (super_task->capacity_addrs));
}

//composite task's inner endTake, replace itself in outer task graph to avoid block.
Error_Code task_dag_graph::spawn_and_goout()
{
    this->end_construct_dag();
#ifdef TEMP_DEBUG
    printf("spawn_and_goout_taskptr=%p\n",cur_super_taskptr);
    fflush(stdout);
#endif

#ifndef CONCURRENT_CONSTRUCT_GRAPH
    _move_successor((aceMesh_task *)cur_super_taskptr, (aceMesh_task*)this->endTask);
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
    if(first_graph_build==0){ 
        graph_build_end=rpcc();
        first_graph_build=1;
        buil_prof[T_GRAPH_BUILD]+=(graph_build_end-graph_build_start);
    }
#endif

#ifdef TEMP_DEBUG
    printf("composite_end_task_affinity=%d,endtask=%p\n",((aceMesh_task*)this->endTask)->task_base.affinity_id,this->endTask);
    fflush(stdout);
#endif
#ifndef CONCURRENT_CONSTRUCT_GRAPH
    this->spawn();
#endif
    this->reset_task_graph();
#ifdef TEMP_DEBUG
    printf("end_of_spawn_and_goout_taskptr=%p\n",cur_super_taskptr);
    fflush(stdout);
#endif
    return ACEMESH_OK;
}

Error_Code task_dag_graph::begin_split_task()
{
#ifdef _DAG_TIME
    if (dag_start_count == 0) {
        dag_start_count = rpcc();
    }
#endif

    this->one_virtual_task_tuples.clear();
#ifdef AUTO_AFFINITY
#ifndef ALL_CYCLIC
    my_affinity.reset_affinity_id();
#endif
//modify_control_thread_num
    //my_affinity.set_thread_num(sche_num_threads);
#endif
    
    return ACEMESH_OK;
}

#define WRITE 1
#define READ  2
int task_dag_graph::update_write_addr(std::map<void*,update_addr_flag>& addr_update, void* addr, int type)
{
    std::map<void*,update_addr_flag>::iterator itr = addr_update.find(addr);
    if(itr == addr_update.end())
    {
        update_addr_flag update_info; 
        if(type == WRITE) 
        {
            update_info.w_flag = true;
        } 
        else if(type == READ) 
        {
            update_info.r_flag = true;
        }
        addr_update.insert( std::map<void*,update_addr_flag>::value_type(addr, update_info) );
    }
    else 
    {
        update_addr_flag& tmp = itr->second; 
        if(type == WRITE && tmp.w_flag == true)
        {
            tmp.w_flag = true;
            return 2;
        }
        if(type == WRITE && tmp.r_flag == true)
        {
            tmp.w_flag = true;
            return 1;
        }
        if(type == READ && tmp.w_flag == true)
        {
            tmp.r_flag = true;
            return 1;
        }

        if(type == WRITE) 
        {
            tmp.w_flag = true;
        } 
        else if(type == READ) 
        {
            tmp.r_flag = true;
        }
    }
    return 0;
}

bool cmp_tuple(const tuple_addr_task& tuple1, const tuple_addr_task& tuple2)
{
    if( tuple1.addr < tuple2.addr ) 
        return true;
    if( tuple1.addr == tuple2.addr && tuple1.t < tuple2.t )
        return true;
    return false;
}

int task_dag_graph::type_add(int type1, int type2)
{
    if(type1 == INOUT || type2 == INOUT)
        return INOUT;
    if(type1 == OUT || type2 == OUT)
    {
        if((type1 == IN && type2 == OUT) ||( type1 == OUT && type2 == IN))
            return INOUT;
        return OUT;
    }
    if (type1 == IN || type2 == IN) 
        return IN;
    else 
        return INOUT_NONE;
}
int task_dag_graph::significance_add(int significance1, int significance2)
{
    return significance1 && significance2;
}
void task_dag_graph::unique_tuples()
{
    std::vector<tuple_addr_task>::iterator tmp_itr;
    for(std::vector<tuple_addr_task>::iterator itr = this->one_virtual_task_tuples.begin(); 
        itr != this->one_virtual_task_tuples.end(); )
    {
        tmp_itr = itr;
        ++itr;
        for(; itr != this->one_virtual_task_tuples.end(); ++itr)
        {
            if(itr->addr != tmp_itr->addr || itr->t != tmp_itr->t )
            {
                break;
            }
            else 
            {
                itr->type =  type_add(itr->type, tmp_itr->type);
#if !defined(LIGHT_BUILD_GRAPH)
                itr->significance = significance_add(itr->significance, tmp_itr->significance);
#endif

                tmp_itr->addr = NULL;
                tmp_itr->t = NULL;
                tmp_itr = itr;
            }
        }
    }
}
Error_Code task_dag_graph::end_split_task()
{

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    unsigned long add_start,e_start=rpcc();
#endif

#ifdef LOW_COUPLING_GRAPH
    //std::cout<<"befor_update_hashmap_and_collect_dep_according_virtual_task_tuple"<<std::endl;
#ifdef CLUSTERING
   update_hashmap_and_collect_dep_according_virtual_task_tuple_with_clustering();
#else
    update_hashmap_and_collect_dep_according_virtual_task_tuple();
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[UPDATE_HASH]+=(rpcc()-e_start);
    add_start=rpcc();
#endif
    //std::cout<<"befor_construct_dep_according_dep_pair"<<std::endl;
    construct_dep_according_dep_pair();
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[BUILD_RELAT_IN]+=(rpcc()-add_start);
#endif
    //std::cout<<"befor_reset_virtual_data"<<std::endl;
    reset_virtual_data();


#else



#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF1)
    unsigned long i_start=rpcc();
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
    unsigned long f_start=rpcc();
#endif

#ifdef MERGE_SAME_ADDR
    std::sort(this->one_virtual_task_tuples.begin(), this->one_virtual_task_tuples.end(), cmp_tuple);
    this->unique_tuples();
#endif
#ifdef DEBUG_GRAPH
    std::map<void*,update_addr_flag> addr_update;
#endif

#ifndef IMMEDIA_UPDATE
    for(std::vector<tuple_addr_task>::iterator itr = this->one_virtual_task_tuples.begin(); 
        itr != this->one_virtual_task_tuples.end(); ++itr)
    {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF1)
        i_start=rpcc();
#endif
        if(itr->type == INOUT_NONE || itr->addr == NULL || itr->t == NULL)
            continue;
#ifdef DEBUG_GRAPH
        int confilt = 0;
#endif
    
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
        f_start=rpcc();
#endif
        std::tr1::unordered_map<void*, tuple_rw_task>::iterator search_itr  = this->addr_task.find(itr->addr);
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
        find_hash_ntimes2++;
        buil_prof[FIND_HASH2]+=(rpcc()-f_start);
#endif
        if(search_itr == addr_task.end())
        {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
            f_start=rpcc();
#endif
            rw_task_tuple tmp;
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
            buil_prof[UPDATE_DATA]+=(rpcc()-f_start);
#endif
            if(itr->type == OUT || itr->type == INOUT)
            {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                f_start=rpcc();
#endif
                tmp.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                tmp.w_task.significance = itr->significance;
#endif
#ifdef DEBUG_GRAPH
                confilt = this->update_write_addr(addr_update, itr->addr, WRITE);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                buil_prof[UPDATE_DATA]+=(rpcc()-f_start);
#endif
            } 
            else if( itr->type == IN )
            {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                f_start=rpcc();
#endif
                task_significance t_s;
                t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                t_s.significance = itr->significance;
#endif
                tmp.r_tasks.push_back(t_s);
#ifdef DEBUG_GRAPH
                confilt = this->update_write_addr(addr_update, itr->addr, READ);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                buil_prof[UPDATE_DATA]+=(rpcc()-f_start);
#endif
            }
#ifdef TEMP_ASSERT
            else 
            {
                assert(0);
            }
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                f_start=rpcc();
#endif
            this->addr_task[itr->addr] = tmp; 
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                buil_prof[UPDATE_DATA]+=(rpcc()-f_start);
#endif
        }
        else
        {
            if(itr->type == OUT || itr->type == INOUT)
            {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                f_start=rpcc();
#endif
                search_itr->second.w_task.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                search_itr->second.w_task.significance = itr->significance;
#endif
                search_itr->second.r_tasks.clear();
#ifdef DEBUG_GRAPH
                confilt = this->update_write_addr(addr_update, itr->addr, WRITE);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                buil_prof[UPDATE_DATA]+=(rpcc()-f_start);
#endif
            }
            else 
            {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                f_start=rpcc();
#endif
                task_significance t_s;
                t_s.t = itr->t;
#if !defined(LIGHT_BUILD_GRAPH)
                t_s.significance = itr->significance;
#endif
                search_itr->second.r_tasks.push_back(t_s);
#ifdef DEBUG_GRAPH
                confilt = this->update_write_addr(addr_update, itr->addr, READ);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF2)
                buil_prof[UPDATE_DATA]+=(rpcc()-f_start);
#endif
            }
        }
#ifdef DEBUG_GRAPH
        if(confilt)
        {
            std::cout<< " warning: this for_all loop (loop name : " << graph_check.get_last_loop_info() 
                << ") has read and write the same addr(block) " << itr->addr << std::endl;
            std::cout<< (confilt==1 ? "Read in one task, Write in another task" : 
                "Write in one task, Write in another task") << std::endl;
            graph_check.print_stack() ;
            if(search_itr != addr_task.end())
            {
                search_itr->second.r_tasks.clear();
            }
        }
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF1)
        buil_prof[UPDATE_ITEM_IN]+=(rpcc()-i_start);
#endif
    }
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    unsigned long e_end=rpcc();
    buil_prof[UPDATE_HASH]+=(e_end-e_start);
#endif

#endif








#if defined(CONCURRENT_CONSTRUCT_GRAPH) && !defined(FAKE_CONCURRENT_GRAPH)
#ifdef ACEMESH_PROFILING_SUCC
    unsigned long spawn_start=rpcc();
#endif

    if(spawn_num==0)
    {
//#ifdef BLOCK_AGENT
        init_queue_args();
//#endif
     int i;
#ifdef LOCAL_FLAG
  #ifdef _SPAWN
        for(i=0;i<64;i++)
  #else
        for(i=0;i<total_num_threads;i++)
  #endif
        {
  #ifdef TARG_SW5
    #ifdef _CIRCULAR_QUEUE
            h2ldm(init_flag,i,cgid)=1;
    #endif
            h2ldm(is_run,i,cgid)=1;
  #elif TARG_SW9
            my_h2ldm(is_run_addr[i],i,cgid,int)=1;
#ifdef DEBUG
            printf("is_run=%d,%d\n",i,my_h2ldm(is_run_addr[i],i,cgid,int));
#endif
     //printf("is_run");
  #endif
    }
#else
        is_run = 1;
    //printf("start working\n");
#endif
        spawn_num=1;
    }
    this->spawn();
    

#ifdef ACEMESH_PROFILING_SUCC
    all_spawn_slave_task_time[64]+=(rpcc()-spawn_start);
    //printf("...%lu,",all_spawn_slave_task_time[65]);
#endif
    need_spawn_tasks.clear();
/*if(spawn_num==1){
    int i;
#ifdef LOCAL_FLAG
#ifdef _SPAWN
    for(i=0;i<64;i++)
#else
    for(i=0;i<total_num_threads;i++)
#endif
    {
#ifdef TARG_SW5
#ifdef _CIRCULAR_QUEUE
        h2ldm(init_flag,i,cgid)=1;
#endif
        h2ldm(is_run,i,cgid)=1;
#elif TARG_SW9
        my_h2ldm(is_run_addr[i],i,cgid,int)=1;
#ifdef DEBUG
        printf("is_run=%d,%d\n",i,my_h2ldm(is_run_addr[i],i,cgid,int));
#endif
#endif
    }
#else
    is_run = 1;
    //printf("start working\n");
#endif
}*/

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
    buil_prof[CLEAR_NEED_SPAWN]+=(rpcc()-e_end);
#endif

#endif

    return ACEMESH_OK;
}


Error_Code task_dag_graph::get_end_tasks()
{
    aceMesh_task* local_t=NULL;
    //std::cout<<"hash_map: addr_task_size=" << this->addr_task.size() << std::endl;
    for(std::tr1::unordered_map<void*, tuple_rw_task>::iterator itr = this->addr_task.begin(); itr != this->addr_task.end(); ++itr)
    {
        //std::cout<<"key of addr_task_item="<<itr->first<<", w_task="<<itr->second.w_task.t<<", r_task_size="<<itr->second.r_tasks.size()<<std::endl;
        local_t=itr->second.w_task.t;
        if(local_t!=NULL)
        {
            if(local_t->vertical_task==NULL && local_t->successor_count==0)
            {
                //std::tr1::unordered_set<aceMesh_task*>::iterator search_itr = end_tasks.find(local_t);
                //if(search_itr == end_tasks.end())
                //{
                    end_tasks.insert(local_t);
                //}
            }
        }
        for(std::vector<task_significance>::iterator r_task_itr = itr->second.r_tasks.begin(); 
                r_task_itr != itr->second.r_tasks.end(); ++r_task_itr)
        {
            local_t=r_task_itr->t;
            if(local_t->vertical_task==NULL && (local_t->successor_count==0))
            {
                //std::tr1::unordered_set<aceMesh_task*>::iterator search_itr = end_tasks.find(local_t);
                //if(search_itr == end_tasks.end())
                //{ 
                   end_tasks.insert(local_t);
                //}
            }
        }

    }
    //std::cout<<"end_task_size="<<end_tasks.size()<<"; end_task2_size="<<end_tasks2.size()<<std::endl;
    // for(std::tr1::unordered_set<aceMesh_task*>::iterator itr=end_tasks.begin(); itr!=end_tasks.end(); ++itr)
    // {
    //     std::cout<< *itr<<std::endl;
    // }
    // for(std::tr1::unordered_set<aceMesh_task*>::iterator itr=end_tasks2.begin(); itr!=end_tasks2.end(); ++itr)
    // {
    //     std::cout<< *itr<<std::endl;
    // }
}



Error_Code task_dag_graph::end_construct_dag()
{
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_end_construct_dag);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    unsigned long e_start=rpcc();
#endif
#ifdef NO_END_TASK_LIST
    get_end_tasks();
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    //unsigned long e_end=rpcc();
    buil_prof[GET_END_LIST]+=(rpcc()-e_start);
#endif

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    e_start=rpcc();
#endif

    if (cur_super_taskptr) {
        //end_task is aceMesh_task
        endTask = (empty_task*) compsite_end_task_constructor();
        #ifdef CONCURRENT_CONSTRUCT_GRAPH       
        my_mutex_unlock(&(((struct task *)endTask)->successor_lock));
      // my_mutex_unlock(&(((struct task *)endTask)->ref_lock));
        _move_successor((aceMesh_task *)cur_super_taskptr, (aceMesh_task*)this->endTask);
        #endif
    } else {
#ifdef DEBUG_GRAPH 
        endTask = end_task_constructor();
#else
        endTask = empty_task_constructor();
#endif
    }



#ifdef TEMP_DEBUG
    printf("construct_end_task=%p,endtak=%p\n",cur_super_taskptr,endTask);
    fflush(stdout);
#endif
#ifdef CONCURRENT_CONSTRUCT_GRAPH
  //my_mutex_unlock(&(endTask->task_base.ref_lock));   
  //endTask->task_base.state=0;
#ifdef SECONDARY_LOCK
   my_mutex_unlock(&(endTask->task_base.slave_lock));
  //my_mutex_unlock(&(endTask->task_base.slave));
  //my_mutex_unlock(&(endTask->task_base.master));
#endif
 my_mutex_unlock(&(endTask->task_base.successor_lock));
    //set_task_state(0, &(endTask->task_base.over));
 my_mutex_unlock(&(endTask->task_base.over));
  // endTask->task_base.successor_lock=0;
   //endTask->task_base.over=0;
  decrement_ref_count(&(endTask->task_base));
#ifdef USE_COMPOSITE_TASK
    if (cur_super_taskptr) {
        decrement_ref_count(&(endTask->task_base));
    }
#endif
     my_h2ldm(con_src_ptr,total_num_threads-1,cgid,unsigned long)=(unsigned long)endTask;
    asm volatile("memb\n");
    //while(my_h2ldm(dest_tail, total_num_threads - 1, cgid, int)!=my_h2ldm(dest_head, total_num_threads - 1, cgid, int));
    for (std::tr1::unordered_set<aceMesh_task *>::iterator itr = end_tasks.begin();
        itr !=end_tasks.end(); ++itr)
    { 
        //my_mutex_lock(&((*itr)->task_base.finished_lock));
        //if ((*itr)->task_base.state<2)
        //{
        add_end_successor((*itr), (task*)endTask);
        //}
        //my_mutex_unlock(&((*itr)->task_base.finished_lock));
    }
            //atomic_add(val,&(endTask->task_base.state));
    //  std::cout << "endTask ref_count " << endTask->task_base.ref_count_t << std::endl;
    //my_h2ldm()
     int temp_tail = my_h2ldm(end_tail, total_num_threads - 1, cgid, int);
         while ((temp_tail + 1) % BLOCK_SIZE == my_h2ldm(end_head, total_num_threads - 1, cgid, int))
                 ;
                    // printf("main_task:%x\n",&(t->task_base));
         my_h2ldm(con_task_ptr[temp_tail], total_num_threads - 1, cgid, unsigned long) =110;
         my_h2ldm(end_tail, total_num_threads - 1, cgid, int) = (temp_tail + 1) % BLOCK_SIZE;
         asm volatile("memb\n");
    // my_h2ldm(con_src_ptr,total_num_threads-1,cgid,unsigned long)=(unsigned long)endTask;
     while(my_h2ldm(con_src_ptr,total_num_threads-1,cgid,unsigned long));
     asm volatile("memb\n");
    
    #ifdef USE_COMPOSITE_TASK
    if (cur_super_taskptr) {
        int val;
        #ifdef TARG_SW5;
        atomic_add(val,&(endTask->task_base.ref_count_t));
        if (val == -2) {
            spawn_to_id(&(endTask->task_base));
        #else
      my_atomic_add(&(endTask->task_base.ref_count_t));
         val=con_sum_pre;
         if (val == -1) {
        my_spawn_to_id(&(endTask->task_base));
         #endif
          
        }
    }
    #endif
#else
    for(std::tr1::unordered_set<aceMesh_task*>::iterator itr = this->end_tasks.begin(); 
        itr != this->end_tasks.end(); ++itr)
    {
        add_end_successor((*itr), (task*)endTask);
    }
#endif
#ifdef DEBUG_GRAPH
    if(this->g_print_graph)
    {
        this->check_graph(1);
        this->g_print_graph = false;
    }
    else
        this->check_graph();
#endif

#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    //e_end=rpcc();
    buil_prof[JOIN_TASK]+=(rpcc()-e_start);
#endif
if(my_mpi_rank<=0)
    std::cout<<"endTask ref_count "<<endTask->task_base.ref_count_t << std::endl;
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
    return ACEMESH_OK;
}

void task_dag_graph::destroy_task_graph() 
{
#ifdef DEBUG
    std::cout << "before destory_taks_graph\n"; 
#endif
    std::cout << "before destory_taks_graph\n"; 
    aceMesh_task *p_task = NULL;
    for(std::vector<aceMesh_task*>::iterator itr = vector_tasks.begin(); itr != vector_tasks.end(); ++itr)
    {
        p_task= *itr;
        ci_task_destructor((ci_task*)p_task);
    }
  
#ifdef DEBUG_GRAPH
    end_task_destructor(endTask);
#else
    empty_task_destructor(endTask);
#endif
#ifdef DEBUG
    std::cout << "end of destory_task_graph\n";
#endif
#ifdef ACEMESH_SCHEDULER_PROFILING
#ifdef _EXEC_TIME
    if(my_mpi_rank<=0)
      std::cout<< "xxxexec time : " << exec_time << std::endl;
//    printf("exec_time_of_last_graph : %10.7f\n",(float)(exec_count-last_exec_count)/MFREQ);   
 //   last_exec_count=exec_count;
#endif
#endif

#ifdef LDMM
    //clear_paraloops();
#endif
}

void task_dag_graph::reset_task_graph()
{
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    unsigned long e_start=rpcc();
#endif
    need_spawn_tasks.clear();
    addr_task.clear();
    end_tasks.clear();
    vector_tasks.clear(); // not you
#ifdef DEBUG_GRAPH
    task_nums = 0;
#endif

#ifdef SUPPORT_PARTITION
    sep_data.clear();
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    buil_prof[RESET_GRAPH]+=rpcc()-e_start;
#endif
}

#ifdef AUTO_AFFINITY
void task_dag_graph::reset_affinity_id()
{
    my_affinity.reset_affinity_id();
}
#endif

void task_dag_graph::init_thread_num(int thread_num)
{
    this->thread_num = thread_num;
}

#ifndef SPECIFY_END_TASKS
inline void task_dag_graph::add_end_task(aceMesh_task* t)
{
#ifdef NO_END_TASK_LIST
    return; 
#endif
    end_tasks.insert(t);
}

inline void task_dag_graph::del_end_task(aceMesh_task* t)
{
#ifdef NO_END_TASK_LIST
    return; 
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
        unsigned long del_start=rpcc();
#endif
    std::tr1::unordered_set<aceMesh_task*>::iterator itr = end_tasks.find(t);
    if( itr == end_tasks.end() )
    {
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
                    buil_prof[DEL_END_L]+=(rpcc()-del_start);
#endif
        return ;
    }
    end_tasks.erase(itr);
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF)
            buil_prof[DEL_END_L]+=(rpcc()-del_start);
#endif

}
#endif


inline void task_dag_graph::get_neighbour_init() {
    num_neighbs = 0;
}
void task_dag_graph::push_neighbor_addr(void *addr) {

    if (num_neighbs > MAX_NEIGHBORS-1) {
      printf("Too many neighbors:%d\n", num_neighbs);
      exit(0);
    }   
    neighbor_addrs[num_neighbs++]=addr;
#ifdef ACEMESH_SCHEDULER_PROFILING
    max_num_neigh=mymax(max_num_neigh, num_neighbs);
#endif
}



#ifdef DEBUG_GRAPH
Error_Code task_dag_graph::begin_split_task(const std::string& loop_info)
{
    graph_check.add_loop_info(loop_info);

    this->one_virtual_task_tuples.clear();
    
#ifdef AUTO_AFFINITY
#ifndef ALL_CYCLIC
    my_affinity.reset_affinity_id();
#endif
#endif
    return ACEMESH_OK;
}
//only support seq exec
bool task_dag_graph::check_graph()
{
    std::cout<< "acemesh :: begining check\n";
    graph_check.check_graph_bfs(need_spawn_tasks, task_nums);
    return true;

}
//only support seq exec
bool task_dag_graph::check_graph(int print_graph)
{
    std::cout<< "acemesh :: begining check\n";
    graph_check.print_loop_depedence(need_spawn_tasks, task_nums);
    return true;
}
#endif

#ifdef SUPPORT_PARTITION
/// no use
Error_Code task_dag_graph::spawn_and_wait_with_separation()
{
    this->end_construct_dag_separation();
    int i = 0;
    for(std::map<int, std::vector<aceMesh_task*> >::iterator map_itr = sep_data.begin(); map_itr != sep_data.end(); ++map_itr)
    {
        std::vector<aceMesh_task*>& my_sep_data = map_itr->second;
        int id = map_itr->first;
        task* my_part = sep_end_tasks[id];
        my_part->increment_ref_count();
#ifdef _EXEC_TIME
        gettimeofday(&start,NULL); 
#endif
        for(std::vector<aceMesh_task*>::iterator itr = my_sep_data.begin(); itr != my_sep_data.end(); ++itr)
        {
            task::spawn(**itr);
        }
#ifdef ACEMESH_PERFORMANCE
        my_perf.record_start();
#endif
        my_part->wait_for_all();

#ifdef ACEMESH_PERFORMANCE
        my_perf.record_end();
#endif

#ifdef _EXEC_TIME
        gettimeofday(&end,NULL);
        exec_time+=(end.tv_sec-start.tv_sec)+(end.tv_usec-start.tv_usec)/1000000.0; 
#endif
    }


    this->destroy_task_graph();
    this->reset_task_graph();
    return ACEMESH_OK;
}
void task_dag_graph::end_construct_dag_separation()
{
    sep_end_tasks.clear();
    for(std::unordered_set<aceMesh_task*>::iterator itr = this->end_tasks.begin(); 
        itr != this->end_tasks.end(); ++itr)
    {
        int id =  (*itr)->get_part_id() ;
        std::map<int, task*>::iterator itr_find = sep_end_tasks.find(id);
        task* tmp = NULL;
        if(itr_find == sep_end_tasks.end())
        {
#ifdef DEBUG_GRAPH
            tmp = end_task_constructor();
#else
            tmp =  empty_task_constructor();
#endif
            sep_end_tasks.insert(std::pair<int,task*>(id, tmp));
        }
        else 
        {
            tmp = itr_find->second;
        }
        (*itr)->add_end_successor(tmp);
    }
#ifdef DEBUG_GRAPH
    if(this->g_print_graph)
    {
        this->check_graph(1);
        this->g_print_graph = false;
    }
    else
        this->check_graph();
#endif
}


#endif

}
